2024-04-06T13:43:26.356 INFO:root:teuthology version: 0.0.1.dev265+gd7d6e60 2024-04-06T13:43:26.357 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --owner scheduled_teuthology@teuthology --name teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056 --description fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} -- /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/orig.config.yaml 2024-04-06T13:43:26.420 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-06T13:43:26.512 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056 branch: reef description: fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7639056' kernel: client: sha1: distro kdb: true sha1: distro ktype: distro last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi mgrmodules: sequential: - print: Enabling mgr modules - exec: mon.a: - ceph mgr module enable snap_schedule - ceph config set mgr mgr/snap_schedule/allow_m_granularity true - ceph config set mgr mgr/snap_schedule/dump_on_update true name: teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: cephfs: max_mds: 3 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true mds_bal_replicate_threshold: 1 mds_export_ephemeral_random_max: 0.1 osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug cephsqlite: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bluestore block size: 96636764160 bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug ms: 1 debug osd: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10000 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - slow metadata IO - SLOW_OPS - slow request sha1: 052adcadd0534eddb0d73c5afbc083a84cd9dc46 subvols: create: 2 subvol_options: --namespace-isolated --size 25000000000 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} ceph-fuse: client.0: mntopts: - context=system_u:object_r:postgresql_db_t:s0 mount_subvol_num: 0 mountpoint: /tmp/cephfs check-counter: counters: mds: - mds.exported - mds.imported - min: 3 name: mds_cache.dir_update - min: 3 name: mds_cache.dir_update_receipt - mds.root_rsnaps - mds_server.req_mksnap_latency.avgcount - mds_server.req_rmsnap_latency.avgcount install: ceph: flavor: default sha1: 052adcadd0534eddb0d73c5afbc083a84cd9dc46 kclient: client.0: mntopts: - context=system_u:object_r:postgresql_db_t:s0 mount_subvol_num: 1 mountpoint: /tmp/cephfs mntopts: - ms_mode=secure - wsync syntax: v1 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: reef sha1: 800ec6d64a7fa0648ad53edfc6e50b41fc4cd674 owner: scheduled_teuthology@teuthology priority: 930 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 seed: 3144 sha1: 052adcadd0534eddb0d73c5afbc083a84cd9dc46 sleep_before_teardown: 0 subset: 362/512 suite: fs suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_800ec6d64a7fa0648ad53edfc6e50b41fc4cd674/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 800ec6d64a7fa0648ad53edfc6e50b41fc4cd674 syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi090.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdBdDgbEv91YZhuMvZHDVA25oXplN4DZaTR0BcGXPTRTTRwHpuZOLEBiSIb22QDGpB+GE6NufEmYxEUeTEudiA= smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOVFhnVzOckaGR4At43MeDqaE0SvMXg0YOZWW1569JXLfOcYV4LD1S3UuRsg7qYW6mPSdbvhTv4JZTtwwc4i+vE= smithi134.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkp91UjlV6CQ+U6xQnPXdnsnfn6Q9zAvre4oI9Zy5N78VmFK+CIHoQCM6YR831rdVHlu58dd4ol3KjVRIPMr1s= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - sequential: - mgrmodules - kclient: null - exec: mon.a: - ceph fs set cephfs balance_automate false - ceph fs subvolumegroup pin cephfs qa random 0.10 - check-counter: {} - fwd_scrub: scrub_timeout: 900 sleep_between_iterations: 1 - exec: mon.a: - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - date +%s > START_TIME - full_sequential_finally: - exec: mon.a: - date +%s > END_TIME - START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - exec: client.0: - sudo ls -l /tmp/cephfs/ && sudo df -h /tmp/cephfs/ - sudo mkdir -m 755 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres && sudo chown postgres:postgres /tmp/cephfs/postgres - sudo -u postgres -- mkdir -m 700 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres/data - sudo semanage fcontext -a -t postgresql_db_t "/tmp/cephfs/postgres(/.*)?" - sudo grep -i postgresql /etc/selinux/targeted/contexts/files/file_contexts.local - sudo restorecon -R -v /tmp/cephfs/postgres - sudo ls -lZaR /tmp/cephfs/postgres/ - sudo mkdir -p /etc/systemd/system/postgresql.service.d/ && printf '[Service]\nEnvironment=PGDATA=/tmp/cephfs/postgres/data\nEnvironment=PGLOG=/tmp/cephfs/postgres/pgstartup.log\n' | sudo tee /etc/systemd/system/postgresql.service.d/env.conf - sudo -u postgres -- postgresql-setup --initdb - sudo ls -lZaR /tmp/cephfs/postgres/ - sudo systemctl start postgresql - sudo -u postgres -- pgbench -s 32 -i - sudo -u postgres -- pgbench -c 100 -j 4 --progress=5 --time=900 - sudo systemctl stop postgresql - sudo ls -lZaR /tmp/cephfs/postgres/ teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: d7d6e60867cadf19fbfd7655a551ae7d58060757 timestamp: 2024-04-03_22:24:02 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-06T13:43:26.512 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_800ec6d64a7fa0648ad53edfc6e50b41fc4cd674/qa; will attempt to use it 2024-04-06T13:43:26.513 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_800ec6d64a7fa0648ad53edfc6e50b41fc4cd674/qa/tasks 2024-04-06T13:43:26.513 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-06T13:43:26.514 INFO:teuthology.task.internal:Checking packages... 2024-04-06T13:43:26.538 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '052adcadd0534eddb0d73c5afbc083a84cd9dc46' 2024-04-06T13:43:26.538 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-06T13:43:26.538 INFO:teuthology.packaging:ref: None 2024-04-06T13:43:26.538 INFO:teuthology.packaging:tag: None 2024-04-06T13:43:26.538 INFO:teuthology.packaging:branch: reef 2024-04-06T13:43:26.538 INFO:teuthology.packaging:sha1: 052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:43:26.538 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2024-04-06T13:43:26.824 INFO:teuthology.task.internal:Found packages for ceph version 18.2.2-1169.g37f5b3f8 2024-04-06T13:43:26.826 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-06T13:43:26.832 INFO:teuthology.task.internal:no buildpackages task found 2024-04-06T13:43:26.832 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-06T13:43:26.855 INFO:teuthology.task.internal:Saving configuration 2024-04-06T13:43:26.878 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-06T13:43:26.884 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-06T13:43:26.910 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi090.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.883055', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdBdDgbEv91YZhuMvZHDVA25oXplN4DZaTR0BcGXPTRTTRwHpuZOLEBiSIb22QDGpB+GE6NufEmYxEUeTEudiA='} 2024-04-06T13:43:26.931 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.886477', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOVFhnVzOckaGR4At43MeDqaE0SvMXg0YOZWW1569JXLfOcYV4LD1S3UuRsg7qYW6mPSdbvhTv4JZTtwwc4i+vE='} 2024-04-06T13:43:26.956 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.884901', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkp91UjlV6CQ+U6xQnPXdnsnfn6Q9zAvre4oI9Zy5N78VmFK+CIHoQCM6YR831rdVHlu58dd4ol3KjVRIPMr1s='} 2024-04-06T13:43:26.956 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-06T13:43:26.963 INFO:teuthology.task.internal:roles: ubuntu@smithi090.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2024-04-06T13:43:26.963 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2024-04-06T13:43:26.963 INFO:teuthology.task.internal:roles: ubuntu@smithi134.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2024-04-06T13:43:26.963 INFO:teuthology.run_tasks:Running task console_log... 2024-04-06T13:43:27.070 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f780faf1160>, signals=[15]) 2024-04-06T13:43:27.071 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-06T13:43:27.088 INFO:teuthology.task.internal:Opening connections... 2024-04-06T13:43:27.088 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T13:43:27.090 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:43:27.178 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-04-06T13:43:27.179 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:43:27.262 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi134.front.sepia.ceph.com 2024-04-06T13:43:27.263 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:43:27.347 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-06T13:43:27.366 DEBUG:teuthology.orchestra.run.smithi090:> uname -m 2024-04-06T13:43:27.390 INFO:teuthology.orchestra.run.smithi090.stdout:x86_64 2024-04-06T13:43:27.390 DEBUG:teuthology.orchestra.run.smithi090:> cat /etc/os-release 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:NAME="Red Hat Enterprise Linux" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:VERSION="8.6 (Ootpa)" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:ID="rhel" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:ID_LIKE="fedora" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:VERSION_ID="8.6" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:PLATFORM_ID="platform:el8" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:ANSI_COLOR="0;31" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:HOME_URL="https://www.redhat.com/" 2024-04-06T13:43:27.454 INFO:teuthology.orchestra.run.smithi090.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-06T13:43:27.455 INFO:teuthology.orchestra.run.smithi090.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-06T13:43:27.455 INFO:teuthology.lock.ops:Updating smithi090.front.sepia.ceph.com on lock server 2024-04-06T13:43:27.479 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-04-06T13:43:27.500 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-04-06T13:43:27.501 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="Red Hat Enterprise Linux" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="8.6 (Ootpa)" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:ID="rhel" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="fedora" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="8.6" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el8" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-06T13:43:27.564 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://www.redhat.com/" 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-06T13:43:27.565 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-06T13:43:27.565 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-04-06T13:43:27.586 DEBUG:teuthology.orchestra.run.smithi134:> uname -m 2024-04-06T13:43:27.608 INFO:teuthology.orchestra.run.smithi134.stdout:x86_64 2024-04-06T13:43:27.608 DEBUG:teuthology.orchestra.run.smithi134:> cat /etc/os-release 2024-04-06T13:43:27.670 INFO:teuthology.orchestra.run.smithi134.stdout:NAME="Red Hat Enterprise Linux" 2024-04-06T13:43:27.670 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION="8.6 (Ootpa)" 2024-04-06T13:43:27.670 INFO:teuthology.orchestra.run.smithi134.stdout:ID="rhel" 2024-04-06T13:43:27.670 INFO:teuthology.orchestra.run.smithi134.stdout:ID_LIKE="fedora" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:VERSION_ID="8.6" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:PLATFORM_ID="platform:el8" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:ANSI_COLOR="0;31" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:HOME_URL="https://www.redhat.com/" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2024-04-06T13:43:27.671 INFO:teuthology.orchestra.run.smithi134.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2024-04-06T13:43:27.672 INFO:teuthology.lock.ops:Updating smithi134.front.sepia.ceph.com on lock server 2024-04-06T13:43:27.694 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-06T13:43:27.839 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-06T13:43:27.857 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-06T13:43:27.857 DEBUG:teuthology.orchestra.run.smithi090:> test '!' -e /home/ubuntu/cephtest 2024-04-06T13:43:27.860 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-04-06T13:43:27.862 DEBUG:teuthology.orchestra.run.smithi134:> test '!' -e /home/ubuntu/cephtest 2024-04-06T13:43:27.881 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-06T13:43:27.887 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-06T13:43:27.887 DEBUG:teuthology.orchestra.run.smithi090:> test -z $(ls -A /var/lib/ceph) 2024-04-06T13:43:27.920 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-04-06T13:43:27.922 DEBUG:teuthology.orchestra.run.smithi134:> test -z $(ls -A /var/lib/ceph) 2024-04-06T13:43:27.957 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-06T13:43:28.037 INFO:teuthology.run_tasks:Running task kernel... 2024-04-06T13:43:28.052 INFO:teuthology.task.kernel:normalize config orig: {'client': {'sha1': 'distro'}, 'kdb': True, 'sha1': 'distro'} 2024-04-06T13:43:28.053 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'sha1': 'distro'}} 2024-04-06T13:43:28.053 INFO:teuthology.task.kernel:config {'client.0': {'sha1': 'distro'}}, timeout 300 2024-04-06T13:43:28.053 DEBUG:teuthology.orchestra.run.smithi090:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-06T13:43:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:28.072 DEBUG:teuthology.orchestra.run.smithi090:> uname -r 2024-04-06T13:43:28.131 INFO:teuthology.orchestra.run.smithi090.stdout:4.18.0-513.9.1.el8_9.x86_64 2024-04-06T13:43:28.131 INFO:teuthology.task.kernel:Running kernel on smithi090: 4.18.0-513.9.1.el8_9.x86_64 2024-04-06T13:43:28.132 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum install -y kernel 2024-04-06T13:43:29.972 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:43:29.972 INFO:teuthology.orchestra.run.smithi090.stdout:Unable to read consumer identity 2024-04-06T13:43:30.277 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:30.277 INFO:teuthology.orchestra.run.smithi090.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-06T13:43:30.277 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:32.847 INFO:teuthology.orchestra.run.smithi090.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2024-04-06T13:43:35.107 INFO:teuthology.orchestra.run.smithi090.stdout:Extra Packages for Enterprise Linux 7.6 MB/s | 16 MB 00:02 2024-04-06T13:43:38.699 INFO:teuthology.orchestra.run.smithi090.stdout:lab-extras 473 kB/s | 24 kB 00:00 2024-04-06T13:43:39.435 INFO:teuthology.orchestra.run.smithi090.stdout:Package kernel-4.18.0-513.9.1.el8_9.x86_64 is already installed. 2024-04-06T13:43:39.435 INFO:teuthology.orchestra.run.smithi090.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2024-04-06T13:43:39.527 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-06T13:43:39.528 INFO:teuthology.orchestra.run.smithi090.stdout:Nothing to do. 2024-04-06T13:43:39.528 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-06T13:43:39.667 DEBUG:teuthology.orchestra.run.smithi090:> echo no | sudo yum reinstall kernel || true 2024-04-06T13:43:40.042 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:43:40.042 INFO:teuthology.orchestra.run.smithi090.stdout:Unable to read consumer identity 2024-04-06T13:43:40.235 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:40.235 INFO:teuthology.orchestra.run.smithi090.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-06T13:43:40.235 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:40.546 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:02 ago on Sat 06 Apr 2024 01:43:38 PM UTC. 2024-04-06T13:43:40.618 INFO:teuthology.orchestra.run.smithi090.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-06T13:43:40.623 INFO:teuthology.orchestra.run.smithi090.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-06T13:43:40.699 INFO:teuthology.orchestra.run.smithi090.stderr:Error: No packages marked for reinstall. 2024-04-06T13:43:40.747 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum reinstall -y kernel || true 2024-04-06T13:43:41.080 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:43:41.080 INFO:teuthology.orchestra.run.smithi090.stdout:Unable to read consumer identity 2024-04-06T13:43:41.351 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:41.351 INFO:teuthology.orchestra.run.smithi090.stdout:This system is not registered with an entitlement server. You can use subscription-manager to register. 2024-04-06T13:43:41.351 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:43:41.652 INFO:teuthology.orchestra.run.smithi090.stdout:Last metadata expiration check: 0:00:03 ago on Sat 06 Apr 2024 01:43:38 PM UTC. 2024-04-06T13:43:41.719 INFO:teuthology.orchestra.run.smithi090.stdout:Installed package kernel-4.18.0-513.9.1.el8_9.x86_64 (from rhel-8-for-x86_64-baseos-rpms) not available. 2024-04-06T13:43:41.720 INFO:teuthology.orchestra.run.smithi090.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2024-04-06T13:43:41.751 INFO:teuthology.orchestra.run.smithi090.stderr:Error: No packages marked for reinstall. 2024-04-06T13:43:41.803 DEBUG:teuthology.orchestra.run.smithi090:> rpm -q kernel | sort -rV | head -n 1 2024-04-06T13:43:41.976 INFO:teuthology.orchestra.run.smithi090.stdout:kernel-4.18.0-513.9.1.el8_9.x86_64 2024-04-06T13:43:41.977 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-513.9.1.el8_9.x86_64 2024-04-06T13:43:41.977 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-06T13:43:41.977 DEBUG:teuthology.parallel:result is None 2024-04-06T13:43:41.977 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-06T13:43:41.988 INFO:teuthology.task.internal:Creating test directory... 2024-04-06T13:43:41.988 DEBUG:teuthology.orchestra.run.smithi090:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-06T13:43:41.990 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-06T13:43:41.992 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-06T13:43:42.012 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-06T13:43:42.020 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-06T13:43:42.027 INFO:teuthology.task.internal:Creating archive directory... 2024-04-06T13:43:42.027 DEBUG:teuthology.orchestra.run.smithi090:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-06T13:43:42.046 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-06T13:43:42.052 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-06T13:43:42.088 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-06T13:43:42.095 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-06T13:43:42.095 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-06T13:43:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:42.140 DEBUG:teuthology.orchestra.run.smithi134:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-06T13:43:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:42.159 DEBUG:teuthology.orchestra.run.smithi090:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-06T13:43:42.161 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-06T13:43:42.183 DEBUG:teuthology.orchestra.run.smithi134:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-06T13:43:42.213 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.226 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.253 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.268 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.269 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.319 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-06T13:43:42.320 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-06T13:43:42.327 INFO:teuthology.task.internal:Configuring sudo... 2024-04-06T13:43:42.327 DEBUG:teuthology.orchestra.run.smithi090:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-06T13:43:42.330 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-06T13:43:42.332 DEBUG:teuthology.orchestra.run.smithi134:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-06T13:43:42.361 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-06T13:43:42.370 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-06T13:43:42.370 DEBUG:teuthology.orchestra.run.smithi090:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-06T13:43:42.396 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-06T13:43:42.402 DEBUG:teuthology.orchestra.run.smithi134:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-06T13:43:42.420 DEBUG:teuthology.orchestra.run.smithi090:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:42.490 DEBUG:teuthology.orchestra.run.smithi090:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:42.572 DEBUG:teuthology.orchestra.run.smithi090:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:42.616 DEBUG:teuthology.orchestra.run.smithi090:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:42.658 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:43:42.658 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-06T13:43:42.732 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:42.786 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:42.848 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:42.938 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:42.979 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:43:42.979 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-06T13:43:43.049 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:43.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-06T13:43:43.151 DEBUG:teuthology.orchestra.run.smithi134:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:43.240 DEBUG:teuthology.orchestra.run.smithi134:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-06T13:43:43.283 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:43:43.284 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-06T13:43:43.355 DEBUG:teuthology.orchestra.run.smithi090:> sudo service rsyslog restart 2024-04-06T13:43:43.357 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-04-06T13:43:43.360 DEBUG:teuthology.orchestra.run.smithi134:> sudo service rsyslog restart 2024-04-06T13:43:43.406 INFO:teuthology.orchestra.run.smithi090.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T13:43:43.418 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T13:43:43.446 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T13:43:43.906 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-06T13:43:43.919 INFO:teuthology.task.internal:Starting timer... 2024-04-06T13:43:43.919 INFO:teuthology.run_tasks:Running task pcp... 2024-04-06T13:43:43.928 INFO:teuthology.run_tasks:Running task selinux... 2024-04-06T13:43:43.936 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:logrotate_t:s0']} 2024-04-06T13:43:43.937 DEBUG:teuthology.orchestra.run.smithi090:> sudo service auditd rotate 2024-04-06T13:43:44.031 INFO:teuthology.orchestra.run.smithi090.stdout:Rotating logs: 2024-04-06T13:43:44.033 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-04-06T13:43:44.094 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-04-06T13:43:44.096 DEBUG:teuthology.orchestra.run.smithi134:> sudo service auditd rotate 2024-04-06T13:43:44.166 INFO:teuthology.orchestra.run.smithi134.stdout:Rotating logs: 2024-04-06T13:43:44.168 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-06T13:43:44.168 DEBUG:teuthology.orchestra.run.smithi090:> /usr/sbin/getenforce 2024-04-06T13:43:44.190 INFO:teuthology.orchestra.run.smithi090.stdout:Permissive 2024-04-06T13:43:44.190 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-04-06T13:43:44.212 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-04-06T13:43:44.212 DEBUG:teuthology.orchestra.run.smithi134:> /usr/sbin/getenforce 2024-04-06T13:43:44.237 INFO:teuthology.orchestra.run.smithi134.stdout:Permissive 2024-04-06T13:43:44.237 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi090.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi105.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi134.front.sepia.ceph.com': 'permissive'} 2024-04-06T13:43:44.237 DEBUG:teuthology.orchestra.run.smithi090:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T13:43:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:44.266 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T13:43:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:44.294 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T13:43:44.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T13:43:44.321 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-06T13:43:44.321 DEBUG:teuthology.orchestra.run.smithi090:> sudo /usr/sbin/setenforce permissive 2024-04-06T13:43:44.352 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-04-06T13:43:44.415 DEBUG:teuthology.orchestra.run.smithi134:> sudo /usr/sbin/setenforce permissive 2024-04-06T13:43:44.449 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-06T13:43:44.458 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-06T13:43:44.463 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-06T13:43:44.595 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-06T13:43:44.613 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-06T13:43:44.615 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi090.front.sepia.ceph.com,smithi105.front.sepia.ceph.com,smithi134.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-06T13:53:24.056 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi090.front.sepia.ceph.com'), Remote(name='ubuntu@smithi105.front.sepia.ceph.com'), Remote(name='ubuntu@smithi134.front.sepia.ceph.com')] 2024-04-06T13:53:24.081 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-06T13:53:24.082 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi090.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:53:24.156 DEBUG:teuthology.orchestra.run.smithi090:> true 2024-04-06T13:53:24.223 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi090.front.sepia.ceph.com' 2024-04-06T13:53:24.224 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-04-06T13:53:24.224 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:53:24.299 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-04-06T13:53:24.368 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-04-06T13:53:24.368 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-04-06T13:53:24.369 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi134.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-06T13:53:24.447 DEBUG:teuthology.orchestra.run.smithi134:> true 2024-04-06T13:53:24.521 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi134.front.sepia.ceph.com' 2024-04-06T13:53:24.521 INFO:teuthology.run_tasks:Running task clock... 2024-04-06T13:53:24.541 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-06T13:53:24.541 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-06T13:53:24.542 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T13:53:24.544 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-06T13:53:24.544 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T13:53:24.546 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-06T13:53:24.546 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T13:53:24.574 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-06T13:53:24.578 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-06T13:53:24.589 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-06T13:53:24.593 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-06T13:53:24.597 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-06T13:53:24.613 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-06T13:53:24.662 INFO:teuthology.orchestra.run.smithi090.stderr:sudo: ntpd: command not found 2024-04-06T13:53:24.672 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-04-06T13:53:24.674 INFO:teuthology.orchestra.run.smithi090.stdout:506 Cannot talk to daemon 2024-04-06T13:53:24.676 INFO:teuthology.orchestra.run.smithi134.stderr:sudo: ntpd: command not found 2024-04-06T13:53:24.684 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-04-06T13:53:24.688 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-06T13:53:24.689 INFO:teuthology.orchestra.run.smithi134.stdout:506 Cannot talk to daemon 2024-04-06T13:53:24.698 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-06T13:53:24.702 INFO:teuthology.orchestra.run.smithi090.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-06T13:53:24.704 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-06T13:53:24.712 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-06T13:53:24.719 INFO:teuthology.orchestra.run.smithi134.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-06T13:53:24.771 INFO:teuthology.orchestra.run.smithi090.stderr:bash: ntpq: command not found 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:=============================================================================== 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.775 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.778 INFO:teuthology.orchestra.run.smithi105.stderr:bash: ntpq: command not found 2024-04-06T13:53:24.781 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T13:53:24.781 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-04-06T13:53:24.782 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.782 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.782 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.782 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.783 INFO:teuthology.orchestra.run.smithi134.stderr:bash: ntpq: command not found 2024-04-06T13:53:24.790 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T13:53:24.790 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-04-06T13:53:24.790 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.790 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.790 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.791 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T13:53:24.791 INFO:teuthology.run_tasks:Running task pexec... 2024-04-06T13:53:24.800 INFO:teuthology.task.pexec:Executing custom commands... 2024-04-06T13:53:24.801 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T13:53:24.801 DEBUG:teuthology.orchestra.run.smithi090:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-06T13:53:24.801 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi105.front.sepia.ceph.com 2024-04-06T13:53:24.801 DEBUG:teuthology.orchestra.run.smithi105:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-06T13:53:24.802 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi134.front.sepia.ceph.com 2024-04-06T13:53:24.802 DEBUG:teuthology.orchestra.run.smithi134:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-06T13:53:25.192 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:25.198 INFO:teuthology.orchestra.run.smithi105.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:25.208 INFO:teuthology.orchestra.run.smithi134.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:26.874 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 49 kB/s | 4.5 kB 00:00 2024-04-06T13:53:27.000 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 38 kB/s | 4.5 kB 00:00 2024-04-06T13:53:27.077 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 25 kB/s | 4.5 kB 00:00 2024-04-06T13:53:27.523 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 29 kB/s | 4.1 kB 00:00 2024-04-06T13:53:27.614 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 27 kB/s | 4.1 kB 00:00 2024-04-06T13:53:27.746 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 31 kB/s | 4.1 kB 00:00 2024-04-06T13:53:30.355 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout:Resetting modules: 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout: container-tools 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-04-06T13:53:30.356 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-04-06T13:53:30.357 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:53:30.472 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout: Package Architecture Version Repository Size 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout:Resetting modules: 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout: container-tools 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout:================================================================================ 2024-04-06T13:53:30.474 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:53:30.547 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-04-06T13:53:30.548 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-04-06T13:53:30.548 INFO:teuthology.orchestra.run.smithi134.stdout: Package Architecture Version Repository Size 2024-04-06T13:53:30.548 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout:Resetting modules: 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout: container-tools 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout:================================================================================ 2024-04-06T13:53:30.549 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:53:43.409 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-04-06T13:53:43.923 INFO:teuthology.orchestra.run.smithi134.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:45.669 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 36 kB/s | 4.5 kB 00:00 2024-04-06T13:53:45.707 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-04-06T13:53:45.967 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-06T13:53:46.177 INFO:teuthology.orchestra.run.smithi105.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:46.329 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 36 kB/s | 4.1 kB 00:00 2024-04-06T13:53:46.426 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:53:47.921 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 29 kB/s | 4.5 kB 00:00 2024-04-06T13:53:48.232 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 29 kB/s | 4.5 kB 00:00 2024-04-06T13:53:48.624 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 20 kB/s | 4.1 kB 00:00 2024-04-06T13:53:48.826 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 31 kB/s | 4.1 kB 00:00 2024-04-06T13:53:49.084 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-04-06T13:53:49.249 INFO:teuthology.orchestra.run.smithi134.stdout:========================================================================================================== 2024-04-06T13:53:49.249 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-04-06T13:53:49.249 INFO:teuthology.orchestra.run.smithi134.stdout:========================================================================================================== 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout:Installing group/module packages: 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: aardvark-dns x86_64 2:1.7.0-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 1.0 M 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: buildah x86_64 1:1.31.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.8 M 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: cockpit-podman noarch 75-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 739 k 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: crun x86_64 1.8.7-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 239 k 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: netavark x86_64 2:1.7.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: python3-podman noarch 4.6.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 154 k 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: skopeo x86_64 2:1.13.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.1 M 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: toolbox x86_64 0.0.99.4-5.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 2.5 M 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: udica noarch 0.2.6-20.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 49 k 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-04-06T13:53:49.250 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: yajl x86_64 2.1.0-12.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:Installing weak dependencies: 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: criu-libs x86_64 3.18-4.module+el8.9.0+20326+387084d0 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:Installing module profiles: 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: container-tools/common 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:Enabling module streams: 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: container-tools rhel8 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:========================================================================================================== 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout:Install 13 Packages 2024-04-06T13:53:49.251 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:53:49.252 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 26 M 2024-04-06T13:53:49.252 INFO:teuthology.orchestra.run.smithi134.stdout:Installed size: 87 M 2024-04-06T13:53:49.252 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-04-06T13:53:49.415 INFO:teuthology.orchestra.run.smithi134.stdout:(1/13): criu-libs-3.18-4.module+el8.9.0+20326+3 241 kB/s | 39 kB 00:00 2024-04-06T13:53:49.432 INFO:teuthology.orchestra.run.smithi134.stdout:(2/13): python3-pyxdg-0.25-16.el8.noarch.rpm 521 kB/s | 94 kB 00:00 2024-04-06T13:53:49.458 INFO:teuthology.orchestra.run.smithi134.stdout:(3/13): python3-pytoml-0.1.14-5.git7dea353.el8. 121 kB/s | 25 kB 00:00 2024-04-06T13:53:49.533 INFO:teuthology.orchestra.run.smithi134.stdout:(4/13): yajl-2.1.0-12.el8.x86_64.rpm 349 kB/s | 41 kB 00:00 2024-04-06T13:53:49.575 INFO:teuthology.orchestra.run.smithi134.stdout:(5/13): aardvark-dns-1.7.0-1.module+el8.9.0+212 7.1 MB/s | 1.0 MB 00:00 2024-04-06T13:53:49.675 INFO:teuthology.orchestra.run.smithi134.stdout:(6/13): cockpit-podman-75-1.module+el8.9.0+2124 5.1 MB/s | 739 kB 00:00 2024-04-06T13:53:49.692 INFO:teuthology.orchestra.run.smithi134.stdout:(7/13): crun-1.8.7-1.module+el8.9.0+21243+a5865 2.0 MB/s | 239 kB 00:00 2024-04-06T13:53:49.809 INFO:teuthology.orchestra.run.smithi134.stdout:(8/13): buildah-1.31.3-3.module+el8.9.0+21243+a 25 MB/s | 8.8 MB 00:00 2024-04-06T13:53:49.893 INFO:teuthology.orchestra.run.smithi134.stdout:(9/13): python3-podman-4.6.0-2.module+el8.9.0+2 767 kB/s | 154 kB 00:00 2024-04-06T13:53:50.026 INFO:teuthology.orchestra.run.smithi134.stdout:(10/13): netavark-1.7.0-2.module+el8.9.0+21243+ 11 MB/s | 3.7 MB 00:00 2024-04-06T13:53:50.102 INFO:teuthology.orchestra.run.smithi134.stdout:(11/13): toolbox-0.0.99.4-5.module+el8.9.0+2124 12 MB/s | 2.5 MB 00:00 2024-04-06T13:53:50.210 INFO:teuthology.orchestra.run.smithi134.stdout:(12/13): skopeo-1.13.3-3.module+el8.9.0+21243+a 20 MB/s | 8.1 MB 00:00 2024-04-06T13:53:50.227 INFO:teuthology.orchestra.run.smithi134.stdout:(13/13): udica-0.2.6-20.module+el8.9.0+21243+a5 247 kB/s | 49 kB 00:00 2024-04-06T13:53:50.227 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:53:50.228 INFO:teuthology.orchestra.run.smithi134.stdout:Total 26 MB/s | 26 MB 00:00 2024-04-06T13:53:50.510 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-04-06T13:53:50.531 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-04-06T13:53:50.532 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-04-06T13:53:50.762 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-04-06T13:53:50.770 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-04-06T13:53:51.400 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-04-06T13:53:51.454 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-04-06T13:53:51.559 INFO:teuthology.orchestra.run.smithi105.stdout:========================================================================================================== 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout:========================================================================================================== 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout:Installing group/module packages: 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: aardvark-dns x86_64 2:1.7.0-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 1.0 M 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: buildah x86_64 1:1.31.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.8 M 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: cockpit-podman noarch 75-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 739 k 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: crun x86_64 1.8.7-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 239 k 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: netavark x86_64 2:1.7.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: python3-podman noarch 4.6.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 154 k 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: skopeo x86_64 2:1.13.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.1 M 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: toolbox x86_64 0.0.99.4-5.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 2.5 M 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout: udica noarch 0.2.6-20.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 49 k 2024-04-06T13:53:51.560 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: yajl x86_64 2.1.0-12.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:Installing weak dependencies: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: criu-libs x86_64 3.18-4.module+el8.9.0+20326+387084d0 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:Installing module profiles: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: container-tools/common 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:Enabling module streams: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: container-tools rhel8 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:========================================================================================================== 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout:Install 13 Packages 2024-04-06T13:53:51.561 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:53:51.562 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 26 M 2024-04-06T13:53:51.562 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 87 M 2024-04-06T13:53:51.562 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-04-06T13:53:51.618 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-06T13:53:51.707 INFO:teuthology.orchestra.run.smithi105.stdout:(1/13): criu-libs-3.18-4.module+el8.9.0+20326+3 273 kB/s | 39 kB 00:00 2024-04-06T13:53:51.723 INFO:teuthology.orchestra.run.smithi105.stdout:(2/13): python3-pytoml-0.1.14-5.git7dea353.el8. 154 kB/s | 25 kB 00:00 2024-04-06T13:53:51.749 INFO:teuthology.orchestra.run.smithi105.stdout:(3/13): python3-pyxdg-0.25-16.el8.noarch.rpm 504 kB/s | 94 kB 00:00 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout:========================================================================================================== 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout:========================================================================================================== 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout:Installing group/module packages: 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout: aardvark-dns x86_64 2:1.7.0-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 1.0 M 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout: buildah x86_64 1:1.31.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.8 M 2024-04-06T13:53:51.778 INFO:teuthology.orchestra.run.smithi090.stdout: cockpit-podman noarch 75-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 739 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: crun x86_64 1.8.7-1.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 239 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: netavark x86_64 2:1.7.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: python3-podman noarch 4.6.0-2.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 154 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: skopeo x86_64 2:1.13.3-3.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 8.1 M 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: toolbox x86_64 0.0.99.4-5.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 2.5 M 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: udica noarch 0.2.6-20.module+el8.9.0+21243+a586538b rhel-8-for-x86_64-appstream-rpms 49 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: yajl x86_64 2.1.0-12.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout:Installing weak dependencies: 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout: criu-libs x86_64 3.18-4.module+el8.9.0+20326+387084d0 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:53:51.779 INFO:teuthology.orchestra.run.smithi090.stdout:Installing module profiles: 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout: container-tools/common 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling module streams: 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout: container-tools rhel8 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout:========================================================================================================== 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout:Install 13 Packages 2024-04-06T13:53:51.780 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:53:51.781 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 26 M 2024-04-06T13:53:51.781 INFO:teuthology.orchestra.run.smithi090.stdout:Installed size: 87 M 2024-04-06T13:53:51.781 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-06T13:53:51.799 INFO:teuthology.orchestra.run.smithi105.stdout:(4/13): yajl-2.1.0-12.el8.x86_64.rpm 445 kB/s | 41 kB 00:00 2024-04-06T13:53:51.888 INFO:teuthology.orchestra.run.smithi105.stdout:(5/13): aardvark-dns-1.7.0-1.module+el8.9.0+212 6.1 MB/s | 1.0 MB 00:00 2024-04-06T13:53:51.927 INFO:teuthology.orchestra.run.smithi090.stdout:(1/13): python3-pytoml-0.1.14-5.git7dea353.el8. 170 kB/s | 25 kB 00:00 2024-04-06T13:53:51.933 INFO:teuthology.orchestra.run.smithi105.stdout:(6/13): cockpit-podman-75-1.module+el8.9.0+2124 5.4 MB/s | 739 kB 00:00 2024-04-06T13:53:51.952 INFO:teuthology.orchestra.run.smithi090.stdout:(2/13): python3-pyxdg-0.25-16.el8.noarch.rpm 549 kB/s | 94 kB 00:00 2024-04-06T13:53:51.969 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 1/13 2024-04-06T13:53:51.977 INFO:teuthology.orchestra.run.smithi090.stdout:(3/13): criu-libs-3.18-4.module+el8.9.0+20326+3 200 kB/s | 39 kB 00:00 2024-04-06T13:53:52.000 INFO:teuthology.orchestra.run.smithi105.stdout:(7/13): crun-1.8.7-1.module+el8.9.0+21243+a5865 2.1 MB/s | 239 kB 00:00 2024-04-06T13:53:52.053 INFO:teuthology.orchestra.run.smithi090.stdout:(4/13): yajl-2.1.0-12.el8.x86_64.rpm 326 kB/s | 41 kB 00:00 2024-04-06T13:53:52.104 INFO:teuthology.orchestra.run.smithi090.stdout:(5/13): aardvark-dns-1.7.0-1.module+el8.9.0+212 6.6 MB/s | 1.0 MB 00:00 2024-04-06T13:53:52.125 INFO:teuthology.orchestra.run.smithi105.stdout:(8/13): buildah-1.31.3-3.module+el8.9.0+21243+a 23 MB/s | 8.8 MB 00:00 2024-04-06T13:53:52.150 INFO:teuthology.orchestra.run.smithi105.stdout:(9/13): python3-podman-4.6.0-2.module+el8.9.0+2 1.0 MB/s | 154 kB 00:00 2024-04-06T13:53:52.171 INFO:teuthology.orchestra.run.smithi090.stdout:(6/13): cockpit-podman-75-1.module+el8.9.0+2124 6.1 MB/s | 739 kB 00:00 2024-04-06T13:53:52.194 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 2/13 2024-04-06T13:53:52.230 INFO:teuthology.orchestra.run.smithi090.stdout:(7/13): crun-1.8.7-1.module+el8.9.0+21243+a5865 1.9 MB/s | 239 kB 00:00 2024-04-06T13:53:52.243 INFO:teuthology.orchestra.run.smithi105.stdout:(10/13): netavark-1.7.0-2.module+el8.9.0+21243+ 12 MB/s | 3.7 MB 00:00 2024-04-06T13:53:52.318 INFO:teuthology.orchestra.run.smithi105.stdout:(11/13): toolbox-0.0.99.4-5.module+el8.9.0+2124 15 MB/s | 2.5 MB 00:00 2024-04-06T13:53:52.368 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : yajl-2.1.0-12.el8.x86_64 3/13 2024-04-06T13:53:52.376 INFO:teuthology.orchestra.run.smithi105.stdout:(12/13): udica-0.2.6-20.module+el8.9.0+21243+a5 370 kB/s | 49 kB 00:00 2024-04-06T13:53:52.380 INFO:teuthology.orchestra.run.smithi090.stdout:(8/13): buildah-1.31.3-3.module+el8.9.0+21243+a 22 MB/s | 8.8 MB 00:00 2024-04-06T13:53:52.406 INFO:teuthology.orchestra.run.smithi090.stdout:(9/13): python3-podman-4.6.0-2.module+el8.9.0+2 875 kB/s | 154 kB 00:00 2024-04-06T13:53:52.506 INFO:teuthology.orchestra.run.smithi090.stdout:(10/13): netavark-1.7.0-2.module+el8.9.0+21243+ 11 MB/s | 3.7 MB 00:00 2024-04-06T13:53:52.526 INFO:teuthology.orchestra.run.smithi105.stdout:(13/13): skopeo-1.13.3-3.module+el8.9.0+21243+a 20 MB/s | 8.1 MB 00:00 2024-04-06T13:53:52.527 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:53:52.527 INFO:teuthology.orchestra.run.smithi105.stdout:Total 26 MB/s | 26 MB 00:00 2024-04-06T13:53:52.581 INFO:teuthology.orchestra.run.smithi090.stdout:(11/13): toolbox-0.0.99.4-5.module+el8.9.0+2124 14 MB/s | 2.5 MB 00:00 2024-04-06T13:53:52.586 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 4/13 2024-04-06T13:53:52.606 INFO:teuthology.orchestra.run.smithi090.stdout:(12/13): udica-0.2.6-20.module+el8.9.0+21243+a5 493 kB/s | 49 kB 00:00 2024-04-06T13:53:52.731 INFO:teuthology.orchestra.run.smithi090.stdout:(13/13): skopeo-1.13.3-3.module+el8.9.0+21243+a 23 MB/s | 8.1 MB 00:00 2024-04-06T13:53:52.732 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:53:52.732 INFO:teuthology.orchestra.run.smithi090.stdout:Total 27 MB/s | 26 MB 00:00 2024-04-06T13:53:52.798 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 5/13 2024-04-06T13:53:52.809 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-04-06T13:53:52.830 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-04-06T13:53:52.831 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-04-06T13:53:53.010 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-06T13:53:53.025 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 6/13 2024-04-06T13:53:53.030 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-06T13:53:53.031 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-06T13:53:53.060 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-04-06T13:53:53.069 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-04-06T13:53:53.259 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-06T13:53:53.267 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-06T13:53:53.293 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 7/13 2024-04-06T13:53:53.782 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-04-06T13:53:53.960 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-06T13:53:54.163 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:54.265 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 1/13 2024-04-06T13:53:54.399 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:54.460 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 1/13 2024-04-06T13:53:54.471 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 2/13 2024-04-06T13:53:54.648 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : yajl-2.1.0-12.el8.x86_64 3/13 2024-04-06T13:53:54.656 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 2/13 2024-04-06T13:53:54.791 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 4/13 2024-04-06T13:53:54.818 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 10/13 2024-04-06T13:53:54.825 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : yajl-2.1.0-12.el8.x86_64 3/13 2024-04-06T13:53:54.986 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 5/13 2024-04-06T13:53:55.010 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 4/13 2024-04-06T13:53:55.163 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 6/13 2024-04-06T13:53:55.181 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 5/13 2024-04-06T13:53:55.398 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 6/13 2024-04-06T13:53:55.415 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 7/13 2024-04-06T13:53:55.642 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 7/13 2024-04-06T13:53:55.734 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 11/13 2024-04-06T13:53:55.972 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 12/13 2024-04-06T13:53:56.154 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:56.322 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:56.524 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:56.560 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:56.560 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/13 2024-04-06T13:53:56.560 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 2/13 2024-04-06T13:53:56.560 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 3/13 2024-04-06T13:53:56.560 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 4/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 5/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 6/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 7/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 10/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 11/13 2024-04-06T13:53:56.561 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 12/13 2024-04-06T13:53:56.577 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:56.753 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:56.947 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 10/13 2024-04-06T13:53:57.142 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 10/13 2024-04-06T13:53:57.858 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 11/13 2024-04-06T13:53:58.026 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 11/13 2024-04-06T13:53:58.093 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 12/13 2024-04-06T13:53:58.276 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:58.321 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 12/13 2024-04-06T13:53:58.521 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 2/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 3/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 4/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 5/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 6/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 7/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:58.711 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:58.712 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 10/13 2024-04-06T13:53:58.712 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 11/13 2024-04-06T13:53:58.712 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 12/13 2024-04-06T13:53:58.932 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 13/13 2024-04-06T13:53:58.932 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/13 2024-04-06T13:53:58.932 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 2/13 2024-04-06T13:53:58.932 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86 3/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 4/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a58653 5/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x 6/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : cockpit-podman-75-1.module+el8.9.0+21243+a586538b. 7/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 8/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x 9/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-podman-4.6.0-2.module+el8.9.0+21243+a58653 10/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x8 11/13 2024-04-06T13:53:58.933 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x 12/13 2024-04-06T13:54:10.675 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 13/13 2024-04-06T13:54:10.675 INFO:teuthology.orchestra.run.smithi134.stdout:Installed products updated. 2024-04-06T13:54:11.251 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: cockpit-podman-75-1.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: python3-podman-4.6.0-2.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.252 INFO:teuthology.orchestra.run.smithi134.stdout: toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.253 INFO:teuthology.orchestra.run.smithi134.stdout: udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.253 INFO:teuthology.orchestra.run.smithi134.stdout: yajl-2.1.0-12.el8.x86_64 2024-04-06T13:54:11.253 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:54:11.253 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-04-06T13:54:11.366 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 13/13 2024-04-06T13:54:11.367 INFO:teuthology.orchestra.run.smithi090.stdout:Installed products updated. 2024-04-06T13:54:11.397 DEBUG:teuthology.parallel:result is None 2024-04-06T13:54:11.946 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:54:11.946 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-06T13:54:11.946 INFO:teuthology.orchestra.run.smithi090.stdout: aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.946 INFO:teuthology.orchestra.run.smithi090.stdout: buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: cockpit-podman-75-1.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: python3-podman-4.6.0-2.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: yajl-2.1.0-12.el8.x86_64 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:54:11.947 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-06T13:54:12.085 DEBUG:teuthology.parallel:result is None 2024-04-06T13:54:13.861 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarc 13/13 2024-04-06T13:54:13.861 INFO:teuthology.orchestra.run.smithi105.stdout:Installed products updated. 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: aardvark-dns-2:1.7.0-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: buildah-1:1.31.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: cockpit-podman-75-1.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: criu-libs-3.18-4.module+el8.9.0+20326+387084d0.x86_64 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: crun-1.8.7-1.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: netavark-2:1.7.0-2.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: python3-podman-4.6.0-2.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:14.454 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyxdg-0.25-16.el8.noarch 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: skopeo-2:1.13.3-3.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: toolbox-0.0.99.4-5.module+el8.9.0+21243+a586538b.x86_64 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: udica-0.2.6-20.module+el8.9.0+21243+a586538b.noarch 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: yajl-2.1.0-12.el8.x86_64 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:54:14.455 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-04-06T13:54:14.601 DEBUG:teuthology.parallel:result is None 2024-04-06T13:54:14.601 INFO:teuthology.run_tasks:Running task install... 2024-04-06T13:54:14.610 DEBUG:teuthology.task.install:project ceph 2024-04-06T13:54:14.610 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46'}} 2024-04-06T13:54:14.610 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46'} 2024-04-06T13:54:14.610 INFO:teuthology.task.install:Using flavor: default 2024-04-06T13:54:14.616 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-06T13:54:14.616 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']} 2024-04-06T13:54:14.617 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'tag': None, 'wait_for_package': False} 2024-04-06T13:54:14.617 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:54:14.619 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'tag': None, 'wait_for_package': False} 2024-04-06T13:54:14.619 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:54:14.620 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'tag': None, 'wait_for_package': False} 2024-04-06T13:54:14.621 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:54:14.844 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/ 2024-04-06T13:54:14.845 INFO:teuthology.task.install.rpm:Package version is 18.2.2-1086.g052adcad 2024-04-06T13:54:14.851 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/ 2024-04-06T13:54:14.851 INFO:teuthology.task.install.rpm:Package version is 18.2.2-1086.g052adcad 2024-04-06T13:54:14.862 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/ 2024-04-06T13:54:14.862 INFO:teuthology.task.install.rpm:Package version is 18.2.2-1086.g052adcad 2024-04-06T13:54:14.956 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-06T13:54:14.956 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:54:14.957 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-06T13:54:14.963 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-06T13:54:14.963 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:54:14.963 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-06T13:54:14.972 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-06T13:54:14.973 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:54:14.973 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-06T13:54:14.990 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-06T13:54:14.991 DEBUG:teuthology.orchestra.run.smithi090:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/052adcadd0534eddb0d73c5afbc083a84cd9dc46/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-06T13:54:14.995 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-06T13:54:14.995 DEBUG:teuthology.orchestra.run.smithi134:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/052adcadd0534eddb0d73c5afbc083a84cd9dc46/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-06T13:54:15.001 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2024-04-06T13:54:15.001 DEBUG:teuthology.orchestra.run.smithi105:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/052adcadd0534eddb0d73c5afbc083a84cd9dc46/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-06T13:54:15.063 DEBUG:teuthology.orchestra.run.smithi090:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-06T13:54:15.068 DEBUG:teuthology.orchestra.run.smithi134:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-06T13:54:15.069 DEBUG:teuthology.orchestra.run.smithi105:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-06T13:54:15.143 DEBUG:teuthology.orchestra.run.smithi090:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-06T13:54:15.146 DEBUG:teuthology.orchestra.run.smithi105:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-06T13:54:15.149 DEBUG:teuthology.orchestra.run.smithi134:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-06T13:54:15.211 INFO:teuthology.orchestra.run.smithi090.stdout:check_obsoletes = 1 2024-04-06T13:54:15.212 INFO:teuthology.orchestra.run.smithi105.stdout:check_obsoletes = 1 2024-04-06T13:54:15.213 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum clean all 2024-04-06T13:54:15.214 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum clean all 2024-04-06T13:54:15.217 INFO:teuthology.orchestra.run.smithi134.stdout:check_obsoletes = 1 2024-04-06T13:54:15.219 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum clean all 2024-04-06T13:54:15.591 INFO:teuthology.orchestra.run.smithi105.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:15.591 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:15.601 INFO:teuthology.orchestra.run.smithi134.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:17.110 INFO:teuthology.orchestra.run.smithi105.stdout:61 files removed 2024-04-06T13:54:17.112 INFO:teuthology.orchestra.run.smithi090.stdout:61 files removed 2024-04-06T13:54:17.133 INFO:teuthology.orchestra.run.smithi134.stdout:61 files removed 2024-04-06T13:54:17.161 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-06T13:54:17.163 DEBUG:teuthology.orchestra.run.smithi090:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-06T13:54:17.193 DEBUG:teuthology.orchestra.run.smithi134:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2024-04-06T13:54:17.498 INFO:teuthology.orchestra.run.smithi090.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:17.498 INFO:teuthology.orchestra.run.smithi105.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:17.532 INFO:teuthology.orchestra.run.smithi134.stdout:Updating Subscription Management repositories. 2024-04-06T13:54:19.407 INFO:teuthology.orchestra.run.smithi105.stdout:ceph packages for x86_64 268 kB/s | 80 kB 00:00 2024-04-06T13:54:19.451 INFO:teuthology.orchestra.run.smithi134.stdout:ceph packages for x86_64 198 kB/s | 80 kB 00:00 2024-04-06T13:54:19.513 INFO:teuthology.orchestra.run.smithi090.stdout:ceph packages for x86_64 232 kB/s | 80 kB 00:00 2024-04-06T13:54:19.810 INFO:teuthology.orchestra.run.smithi134.stdout:ceph noarch packages 35 kB/s | 11 kB 00:00 2024-04-06T13:54:20.047 INFO:teuthology.orchestra.run.smithi090.stdout:ceph noarch packages 23 kB/s | 11 kB 00:00 2024-04-06T13:54:20.119 INFO:teuthology.orchestra.run.smithi134.stdout:ceph source packages 5.3 kB/s | 1.5 kB 00:00 2024-04-06T13:54:20.191 INFO:teuthology.orchestra.run.smithi105.stdout:ceph noarch packages 15 kB/s | 11 kB 00:00 2024-04-06T13:54:20.295 INFO:teuthology.orchestra.run.smithi134.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2024-04-06T13:54:20.364 INFO:teuthology.orchestra.run.smithi090.stdout:ceph source packages 5.1 kB/s | 1.5 kB 00:00 2024-04-06T13:54:20.515 INFO:teuthology.orchestra.run.smithi090.stdout:Copr repo for python3-asyncssh owned by ceph 28 kB/s | 3.5 kB 00:00 2024-04-06T13:54:20.517 INFO:teuthology.orchestra.run.smithi105.stdout:ceph source packages 5.0 kB/s | 1.5 kB 00:00 2024-04-06T13:54:20.685 INFO:teuthology.orchestra.run.smithi105.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2024-04-06T13:54:21.212 INFO:teuthology.orchestra.run.smithi134.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2024-04-06T13:54:21.846 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2024-04-06T13:54:24.763 INFO:teuthology.orchestra.run.smithi134.stdout:lab-extras 493 kB/s | 24 kB 00:00 2024-04-06T13:54:25.478 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 392 kB/s | 24 kB 00:00 2024-04-06T13:54:25.663 INFO:teuthology.orchestra.run.smithi090.stdout:Extra Packages for Enterprise Linux 3.2 MB/s | 16 MB 00:05 2024-04-06T13:54:26.104 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 47 MB/s | 60 MB 00:01 2024-04-06T13:54:27.235 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 36 MB/s | 60 MB 00:01 2024-04-06T13:54:29.302 INFO:teuthology.orchestra.run.smithi090.stdout:lab-extras 400 kB/s | 24 kB 00:00 2024-04-06T13:54:30.454 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat CodeReady Linux Builder for RHEL 8 x86_ 8.2 MB/s | 8.8 MB 00:01 2024-04-06T13:54:33.462 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 42 MB/s | 60 MB 00:01 2024-04-06T13:54:36.657 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat CodeReady Linux Builder for RHEL 8 x86_ 15 MB/s | 8.8 MB 00:00 2024-04-06T13:54:39.014 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 47 MB/s | 66 MB 00:01 2024-04-06T13:54:39.558 INFO:teuthology.orchestra.run.smithi134.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 51 MB/s | 66 MB 00:01 2024-04-06T13:54:44.784 INFO:teuthology.orchestra.run.smithi090.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 48 MB/s | 66 MB 00:01 2024-04-06T13:54:49.389 INFO:teuthology.orchestra.run.smithi105.stdout:Red Hat CodeReady Linux Builder for RHEL 8 x86_ 13 MB/s | 8.8 MB 00:00 2024-04-06T13:54:54.697 INFO:teuthology.orchestra.run.smithi134.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:54.698 INFO:teuthology.orchestra.run.smithi134.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:54.716 INFO:teuthology.orchestra.run.smithi134.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:54.717 INFO:teuthology.orchestra.run.smithi134.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:54.719 INFO:teuthology.orchestra.run.smithi134.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2024-04-06T13:54:54.720 INFO:teuthology.orchestra.run.smithi134.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2024-04-06T13:54:54.723 INFO:teuthology.orchestra.run.smithi134.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2024-04-06T13:54:54.727 INFO:teuthology.orchestra.run.smithi134.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2024-04-06T13:54:54.728 INFO:teuthology.orchestra.run.smithi134.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2024-04-06T13:54:54.947 INFO:teuthology.orchestra.run.smithi134.stdout:Dependencies resolved. 2024-04-06T13:54:54.961 INFO:teuthology.orchestra.run.smithi134.stdout:=================================================================================================================================== 2024-04-06T13:54:54.961 INFO:teuthology.orchestra.run.smithi134.stdout: Package Arch Version Repository Size 2024-04-06T13:54:54.961 INFO:teuthology.orchestra.run.smithi134.stdout:=================================================================================================================================== 2024-04-06T13:54:54.961 INFO:teuthology.orchestra.run.smithi134.stdout:Installing: 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: NetworkManager-initscripts-updown noarch 1:1.40.16-13.el8_9 rhel-8-for-x86_64-baseos-rpms 144 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.4 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-base x86_64 2:18.2.2-1086.g052adcad.el8 ceph 5.5 M 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 860 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-immutable-object-cache x86_64 2:18.2.2-1086.g052adcad.el8 ceph 165 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr x86_64 2:18.2.2-1086.g052adcad.el8 ceph 1.6 M 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 138 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-dashboard noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 3.5 M 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 7.4 M 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-rook noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 49 k 2024-04-06T13:54:54.962 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-radosgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 13 M 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-test x86_64 2:18.2.2-1086.g052adcad.el8 ceph 47 M 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-volume noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 263 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 173 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: cephfs-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 261 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: cephfs-top noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 18 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libelf-devel x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 62 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 31 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 781 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: librados-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 139 k 2024-04-06T13:54:54.963 INFO:teuthology.orchestra.run.smithi134.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: libuuid-devel x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 100 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: openssl-devel x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 2.3 M 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 1.5 M 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql-contrib x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 812 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql-server x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 5.1 M 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cephfs x86_64 2:18.2.2-1086.g052adcad.el8 ceph 196 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rados x86_64 2:18.2.2-1086.g052adcad.el8 ceph 370 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 365 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 107 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 95 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.5 M 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-nbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 182 k 2024-04-06T13:54:54.964 INFO:teuthology.orchestra.run.smithi134.stdout: xfsprogs-devel x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 73 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout:Upgrading: 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libelf x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 232 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libs x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 303 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: iproute x86_64 6.2.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 854 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: libblkid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: libfdisk x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 254 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: libmount x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 237 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: librados2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.7 M 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: librbd1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.6 M 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: libsmartcols x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 179 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: libuuid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 99 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: openssl x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 711 k 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: openssl-libs x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 1.5 M 2024-04-06T13:54:54.965 INFO:teuthology.orchestra.run.smithi134.stdout: util-linux x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 2.5 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: util-linux-user x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 103 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: xfsdump x86_64 3.1.8-7.el8_9 rhel-8-for-x86_64-baseos-rpms 333 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: xfsprogs x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 1.1 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout:Installing dependencies: 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: automake noarch 1.16.1-8.el8 rhel-8-for-x86_64-appstream-rpms 714 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 22 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-grafana-dashboards noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 23 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mds x86_64 2:18.2.2-1086.g052adcad.el8 ceph 2.3 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-modules-core noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 245 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mon x86_64 2:18.2.2-1086.g052adcad.el8 ceph 4.8 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-osd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 18 M 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-prometheus-alerts noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 15 k 2024-04-06T13:54:54.966 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-selinux x86_64 2:18.2.2-1086.g052adcad.el8 ceph 24 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: fuse x86_64 2.9.7-17.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: jq x86_64 1.6-7.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: krb5-devel x86_64 1.18.2-26.el8_9 rhel-8-for-x86_64-baseos-rpms 562 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libcephsqlite x86_64 2:18.2.2-1086.g052adcad.el8 ceph 142 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libgfortran x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 645 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libpq x86_64 13.11-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libquadmath x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 172 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: librabbitmq x86_64 0.9.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 47 k 2024-04-06T13:54:54.967 INFO:teuthology.orchestra.run.smithi134.stdout: libradosstriper1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 475 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: librgw2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.5 M 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: libzstd-devel x86_64 1.4.4-1.el8 rhel-8-for-x86_64-baseos-rpms 44 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: oniguruma x86_64 6.8.2-2.1.el8_9 rhel-8-for-x86_64-appstream-rpms 188 k 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: openblas x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.0 M 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-threads x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.2 M 2024-04-06T13:54:54.968 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-argparse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 45 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 129 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2024-04-06T13:54:54.969 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cffi x86_64 1.11.5-6.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cryptography x86_64 3.2.1-7.el8_9 rhel-8-for-x86_64-baseos-rpms 559 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2024-04-06T13:54:54.970 INFO:teuthology.orchestra.run.smithi134.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-mako noarch 1.0.6-14.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-packaging noarch 16.8-9.el8 codeready-builder-for-rhel-8-x86_64-rpms 48 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2024-04-06T13:54:54.971 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytz noarch 2017.2-11.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-scipy x86_64 1.0.0-21.module+el8.9.0+20976+d3c38525 rhel-8-for-x86_64-appstream-rpms 14 M 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2024-04-06T13:54:54.972 INFO:teuthology.orchestra.run.smithi134.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2024-04-06T13:54:54.973 INFO:teuthology.orchestra.run.smithi134.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout:Enabling module streams: 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql 10 2024-04-06T13:54:54.974 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:54:54.975 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction Summary 2024-04-06T13:54:54.975 INFO:teuthology.orchestra.run.smithi134.stdout:=================================================================================================================================== 2024-04-06T13:54:54.975 INFO:teuthology.orchestra.run.smithi134.stdout:Install 130 Packages 2024-04-06T13:54:54.975 INFO:teuthology.orchestra.run.smithi134.stdout:Upgrade 16 Packages 2024-04-06T13:54:54.975 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:54:54.976 INFO:teuthology.orchestra.run.smithi134.stdout:Total download size: 215 M 2024-04-06T13:54:54.976 INFO:teuthology.orchestra.run.smithi134.stdout:Downloading Packages: 2024-04-06T13:54:55.454 INFO:teuthology.orchestra.run.smithi134.stdout:(1/146): ceph-18.2.2-1086.g052adcad.el8.x86_64. 46 kB/s | 6.4 kB 00:00 2024-04-06T13:54:55.808 INFO:teuthology.orchestra.run.smithi134.stdout:(2/146): ceph-base-18.2.2-1086.g052adcad.el8.x8 11 MB/s | 5.5 MB 00:00 2024-04-06T13:54:55.842 INFO:teuthology.orchestra.run.smithi134.stdout:(3/146): ceph-fuse-18.2.2-1086.g052adcad.el8.x8 2.2 MB/s | 860 kB 00:00 2024-04-06T13:54:55.867 INFO:teuthology.orchestra.run.smithi134.stdout:(4/146): ceph-immutable-object-cache-18.2.2-108 2.7 MB/s | 165 kB 00:00 2024-04-06T13:54:55.984 INFO:teuthology.orchestra.run.smithi134.stdout:(5/146): ceph-mds-18.2.2-1086.g052adcad.el8.x86 16 MB/s | 2.3 MB 00:00 2024-04-06T13:54:56.259 INFO:teuthology.orchestra.run.smithi134.stdout:(6/146): ceph-mon-18.2.2-1086.g052adcad.el8.x86 18 MB/s | 4.8 MB 00:00 2024-04-06T13:54:56.301 INFO:teuthology.orchestra.run.smithi134.stdout:(7/146): ceph-mgr-18.2.2-1086.g052adcad.el8.x86 3.6 MB/s | 1.6 MB 00:00 2024-04-06T13:54:56.446 INFO:teuthology.orchestra.run.smithi105.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:56.447 INFO:teuthology.orchestra.run.smithi105.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:56.466 INFO:teuthology.orchestra.run.smithi105.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:56.467 INFO:teuthology.orchestra.run.smithi105.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:56.468 INFO:teuthology.orchestra.run.smithi105.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2024-04-06T13:54:56.470 INFO:teuthology.orchestra.run.smithi105.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2024-04-06T13:54:56.473 INFO:teuthology.orchestra.run.smithi105.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2024-04-06T13:54:56.477 INFO:teuthology.orchestra.run.smithi105.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2024-04-06T13:54:56.478 INFO:teuthology.orchestra.run.smithi105.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2024-04-06T13:54:56.704 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-04-06T13:54:56.718 INFO:teuthology.orchestra.run.smithi105.stdout:=================================================================================================================================== 2024-04-06T13:54:56.718 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout:=================================================================================================================================== 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: NetworkManager-initscripts-updown noarch 1:1.40.16-13.el8_9 rhel-8-for-x86_64-baseos-rpms 144 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.4 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base x86_64 2:18.2.2-1086.g052adcad.el8 ceph 5.5 M 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 860 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache x86_64 2:18.2.2-1086.g052adcad.el8 ceph 165 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr x86_64 2:18.2.2-1086.g052adcad.el8 ceph 1.6 M 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 138 k 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-dashboard noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 3.5 M 2024-04-06T13:54:56.719 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 7.4 M 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-rook noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 49 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 13 M 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test x86_64 2:18.2.2-1086.g052adcad.el8 ceph 47 M 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-volume noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 263 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 173 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: cephfs-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 261 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: cephfs-top noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 18 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libelf-devel x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 62 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 31 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 781 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 139 k 2024-04-06T13:54:56.720 INFO:teuthology.orchestra.run.smithi105.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: libuuid-devel x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 100 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: openssl-devel x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 2.3 M 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 1.5 M 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql-contrib x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 812 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql-server x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 5.1 M 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs x86_64 2:18.2.2-1086.g052adcad.el8 ceph 196 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados x86_64 2:18.2.2-1086.g052adcad.el8 ceph 370 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 365 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 107 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 95 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.5 M 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 182 k 2024-04-06T13:54:56.721 INFO:teuthology.orchestra.run.smithi105.stdout: xfsprogs-devel x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 73 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrading: 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libelf x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 232 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libs x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 303 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: iproute x86_64 6.2.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 854 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: libblkid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: libfdisk x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 254 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: libmount x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 237 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: librados2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.7 M 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.6 M 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: libsmartcols x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 179 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: libuuid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 99 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: openssl x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 711 k 2024-04-06T13:54:56.722 INFO:teuthology.orchestra.run.smithi105.stdout: openssl-libs x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 1.5 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: util-linux x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 2.5 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: util-linux-user x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 103 k 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: xfsdump x86_64 3.1.8-7.el8_9 rhel-8-for-x86_64-baseos-rpms 333 k 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: xfsprogs x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 1.1 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: automake noarch 1.16.1-8.el8 rhel-8-for-x86_64-appstream-rpms 714 k 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 22 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-grafana-dashboards noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 23 k 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds x86_64 2:18.2.2-1086.g052adcad.el8 ceph 2.3 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 245 k 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon x86_64 2:18.2.2-1086.g052adcad.el8 ceph 4.8 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 18 M 2024-04-06T13:54:56.723 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-prometheus-alerts noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 15 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux x86_64 2:18.2.2-1086.g052adcad.el8 ceph 24 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: fuse x86_64 2.9.7-17.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: jq x86_64 1.6-7.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: krb5-devel x86_64 1.18.2-26.el8_9 rhel-8-for-x86_64-baseos-rpms 562 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite x86_64 2:18.2.2-1086.g052adcad.el8 ceph 142 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: libgfortran x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 645 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: libpq x86_64 13.11-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: libquadmath x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 172 k 2024-04-06T13:54:56.724 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq x86_64 0.9.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 47 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 475 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.5 M 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: libzstd-devel x86_64 1.4.4-1.el8 rhel-8-for-x86_64-baseos-rpms 44 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: oniguruma x86_64 6.8.2-2.1.el8_9 rhel-8-for-x86_64-appstream-rpms 188 k 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: openblas x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.0 M 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-threads x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.2 M 2024-04-06T13:54:56.725 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 45 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 129 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi x86_64 1.11.5-6.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2024-04-06T13:54:56.726 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography x86_64 3.2.1-7.el8_9 rhel-8-for-x86_64-baseos-rpms 559 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako noarch 1.0.6-14.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:54:56.727 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging noarch 16.8-9.el8 codeready-builder-for-rhel-8-x86_64-rpms 48 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2024-04-06T13:54:56.728 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz noarch 2017.2-11.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-scipy x86_64 1.0.0-21.module+el8.9.0+20976+d3c38525 rhel-8-for-x86_64-appstream-rpms 14 M 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2024-04-06T13:54:56.729 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout:Enabling module streams: 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql 10 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout:=================================================================================================================================== 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout:Install 130 Packages 2024-04-06T13:54:56.730 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrade 16 Packages 2024-04-06T13:54:56.731 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:54:56.731 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 215 M 2024-04-06T13:54:56.731 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-04-06T13:54:57.124 INFO:teuthology.orchestra.run.smithi105.stdout:(1/146): ceph-18.2.2-1086.g052adcad.el8.x86_64. 20 kB/s | 6.4 kB 00:00 2024-04-06T13:54:57.277 INFO:teuthology.orchestra.run.smithi134.stdout:(8/146): ceph-osd-18.2.2-1086.g052adcad.el8.x86 18 MB/s | 18 MB 00:01 2024-04-06T13:54:57.369 INFO:teuthology.orchestra.run.smithi134.stdout:(9/146): ceph-selinux-18.2.2-1086.g052adcad.el8 265 kB/s | 24 kB 00:00 2024-04-06T13:54:57.866 INFO:teuthology.orchestra.run.smithi105.stdout:(2/146): ceph-fuse-18.2.2-1086.g052adcad.el8.x8 1.1 MB/s | 860 kB 00:00 2024-04-06T13:54:58.058 INFO:teuthology.orchestra.run.smithi105.stdout:(3/146): ceph-immutable-object-cache-18.2.2-108 865 kB/s | 165 kB 00:00 2024-04-06T13:54:58.652 INFO:teuthology.orchestra.run.smithi134.stdout:(10/146): ceph-radosgw-18.2.2-1086.g052adcad.el 5.6 MB/s | 13 MB 00:02 2024-04-06T13:54:58.752 INFO:teuthology.orchestra.run.smithi134.stdout:(11/146): cephfs-mirror-18.2.2-1086.g052adcad.e 2.6 MB/s | 261 kB 00:00 2024-04-06T13:54:58.844 INFO:teuthology.orchestra.run.smithi134.stdout:(12/146): libcephfs-devel-18.2.2-1086.g052adcad 338 kB/s | 31 kB 00:00 2024-04-06T13:54:58.874 INFO:teuthology.orchestra.run.smithi105.stdout:(4/146): ceph-base-18.2.2-1086.g052adcad.el8.x8 2.7 MB/s | 5.5 MB 00:02 2024-04-06T13:54:59.069 INFO:teuthology.orchestra.run.smithi134.stdout:(13/146): libcephfs2-18.2.2-1086.g052adcad.el8. 3.4 MB/s | 781 kB 00:00 2024-04-06T13:54:59.178 INFO:teuthology.orchestra.run.smithi134.stdout:(14/146): libcephsqlite-18.2.2-1086.g052adcad.e 1.3 MB/s | 142 kB 00:00 2024-04-06T13:54:59.241 INFO:teuthology.orchestra.run.smithi105.stdout:(5/146): ceph-mgr-18.2.2-1086.g052adcad.el8.x86 4.3 MB/s | 1.6 MB 00:00 2024-04-06T13:54:59.262 INFO:teuthology.orchestra.run.smithi134.stdout:(15/146): librados-devel-18.2.2-1086.g052adcad. 1.6 MB/s | 139 kB 00:00 2024-04-06T13:54:59.375 INFO:teuthology.orchestra.run.smithi105.stdout:(6/146): ceph-mds-18.2.2-1086.g052adcad.el8.x86 1.7 MB/s | 2.3 MB 00:01 2024-04-06T13:54:59.503 INFO:teuthology.orchestra.run.smithi134.stdout:(16/146): libradosstriper1-18.2.2-1086.g052adca 1.9 MB/s | 475 kB 00:00 2024-04-06T13:54:59.559 INFO:teuthology.orchestra.run.smithi090.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:59.560 INFO:teuthology.orchestra.run.smithi090.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2024-04-06T13:54:59.579 INFO:teuthology.orchestra.run.smithi090.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:59.580 INFO:teuthology.orchestra.run.smithi090.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2024-04-06T13:54:59.582 INFO:teuthology.orchestra.run.smithi090.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2024-04-06T13:54:59.583 INFO:teuthology.orchestra.run.smithi090.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2024-04-06T13:54:59.586 INFO:teuthology.orchestra.run.smithi090.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2024-04-06T13:54:59.590 INFO:teuthology.orchestra.run.smithi090.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2024-04-06T13:54:59.592 INFO:teuthology.orchestra.run.smithi090.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2024-04-06T13:54:59.684 INFO:teuthology.orchestra.run.smithi105.stdout:(7/146): ceph-common-18.2.2-1086.g052adcad.el8. 7.5 MB/s | 22 MB 00:02 2024-04-06T13:54:59.809 INFO:teuthology.orchestra.run.smithi090.stdout:Dependencies resolved. 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout:=================================================================================================================================== 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: Package Arch Version Repository Size 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout:=================================================================================================================================== 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout:Installing: 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: NetworkManager-initscripts-updown noarch 1:1.40.16-13.el8_9 rhel-8-for-x86_64-baseos-rpms 144 k 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: ceph x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.4 k 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-base x86_64 2:18.2.2-1086.g052adcad.el8 ceph 5.5 M 2024-04-06T13:54:59.824 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 860 k 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-immutable-object-cache x86_64 2:18.2.2-1086.g052adcad.el8 ceph 165 k 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr x86_64 2:18.2.2-1086.g052adcad.el8 ceph 1.6 M 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 138 k 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-dashboard noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 3.5 M 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 7.4 M 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-rook noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 49 k 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-radosgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 13 M 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-test x86_64 2:18.2.2-1086.g052adcad.el8 ceph 47 M 2024-04-06T13:54:59.825 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-volume noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 263 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: cephadm noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 173 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: cephfs-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 261 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: cephfs-top noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 18 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libelf-devel x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 62 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 31 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 781 k 2024-04-06T13:54:59.826 INFO:teuthology.orchestra.run.smithi090.stdout: librados-devel x86_64 2:18.2.2-1086.g052adcad.el8 ceph 139 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: libuuid-devel x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 100 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: openssl-devel x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 2.3 M 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 1.5 M 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql-contrib x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 812 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql-server x86_64 10.23-4.module+el8.9.0+21317+d2c6bf8a rhel-8-for-x86_64-appstream-rpms 5.1 M 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cephfs x86_64 2:18.2.2-1086.g052adcad.el8 ceph 196 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rados x86_64 2:18.2.2-1086.g052adcad.el8 ceph 370 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 365 k 2024-04-06T13:54:59.827 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rgw x86_64 2:18.2.2-1086.g052adcad.el8 ceph 107 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-fuse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 95 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-mirror x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.5 M 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-nbd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 182 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: xfsprogs-devel x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 73 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout:Upgrading: 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libelf x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 232 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libs x86_64 0.189-3.el8 rhel-8-for-x86_64-baseos-rpms 303 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: iproute x86_64 6.2.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 854 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: libblkid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:59.828 INFO:teuthology.orchestra.run.smithi090.stdout: libfdisk x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 254 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: libmount x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 237 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: librados2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.7 M 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: librbd1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 3.6 M 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: libsmartcols x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 179 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: libuuid x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 99 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: openssl x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 711 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: openssl-libs x86_64 1:1.1.1k-12.el8_9 rhel-8-for-x86_64-baseos-rpms 1.5 M 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: util-linux x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 2.5 M 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: util-linux-user x86_64 2.32.1-43.el8 rhel-8-for-x86_64-baseos-rpms 103 k 2024-04-06T13:54:59.829 INFO:teuthology.orchestra.run.smithi090.stdout: xfsdump x86_64 3.1.8-7.el8_9 rhel-8-for-x86_64-baseos-rpms 333 k 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: xfsprogs x86_64 5.0.0-12.el8 rhel-8-for-x86_64-baseos-rpms 1.1 M 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout:Installing dependencies: 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: automake noarch 1.16.1-8.el8 rhel-8-for-x86_64-appstream-rpms 714 k 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 22 M 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-grafana-dashboards noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 23 k 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mds x86_64 2:18.2.2-1086.g052adcad.el8 ceph 2.3 M 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-modules-core noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 245 k 2024-04-06T13:54:59.830 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mon x86_64 2:18.2.2-1086.g052adcad.el8 ceph 4.8 M 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-osd x86_64 2:18.2.2-1086.g052adcad.el8 ceph 18 M 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-prometheus-alerts noarch 2:18.2.2-1086.g052adcad.el8 ceph-noarch 15 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-selinux x86_64 2:18.2.2-1086.g052adcad.el8 ceph 24 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: fuse x86_64 2.9.7-17.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: jq x86_64 1.6-7.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2024-04-06T13:54:59.831 INFO:teuthology.orchestra.run.smithi090.stdout: krb5-devel x86_64 1.18.2-26.el8_9 rhel-8-for-x86_64-baseos-rpms 562 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libcephsqlite x86_64 2:18.2.2-1086.g052adcad.el8 ceph 142 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libgfortran x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 645 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libpq x86_64 13.11-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libquadmath x86_64 8.5.0-20.el8 rhel-8-for-x86_64-baseos-rpms 172 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: librabbitmq x86_64 0.9.0-5.el8_9 rhel-8-for-x86_64-baseos-rpms 47 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: libradosstriper1 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 475 k 2024-04-06T13:54:59.832 INFO:teuthology.orchestra.run.smithi090.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: librgw2 x86_64 2:18.2.2-1086.g052adcad.el8 ceph 6.5 M 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: libzstd-devel x86_64 1.4.4-1.el8 rhel-8-for-x86_64-baseos-rpms 44 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: oniguruma x86_64 6.8.2-2.1.el8_9 rhel-8-for-x86_64-appstream-rpms 188 k 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: openblas x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.0 M 2024-04-06T13:54:59.833 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-threads x86_64 0.3.15-6.el8 rhel-8-for-x86_64-appstream-rpms 5.2 M 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2024-04-06T13:54:59.834 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-argparse x86_64 2:18.2.2-1086.g052adcad.el8 ceph 45 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-common x86_64 2:18.2.2-1086.g052adcad.el8 ceph 129 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cffi x86_64 1.11.5-6.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cryptography x86_64 3.2.1-7.el8_9 rhel-8-for-x86_64-baseos-rpms 559 k 2024-04-06T13:54:59.835 INFO:teuthology.orchestra.run.smithi090.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-mako noarch 1.0.6-14.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2024-04-06T13:54:59.836 INFO:teuthology.orchestra.run.smithi090.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-packaging noarch 16.8-9.el8 codeready-builder-for-rhel-8-x86_64-rpms 48 k 2024-04-06T13:54:59.837 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytz noarch 2017.2-11.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2024-04-06T13:54:59.838 INFO:teuthology.orchestra.run.smithi090.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-scipy x86_64 1.0.0-21.module+el8.9.0+20976+d3c38525 rhel-8-for-x86_64-appstream-rpms 14 M 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2024-04-06T13:54:59.839 INFO:teuthology.orchestra.run.smithi090.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2024-04-06T13:54:59.840 INFO:teuthology.orchestra.run.smithi090.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling module streams: 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql 10 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction Summary 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout:=================================================================================================================================== 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout:Install 130 Packages 2024-04-06T13:54:59.841 INFO:teuthology.orchestra.run.smithi090.stdout:Upgrade 16 Packages 2024-04-06T13:54:59.842 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:54:59.842 INFO:teuthology.orchestra.run.smithi090.stdout:Total download size: 215 M 2024-04-06T13:54:59.843 INFO:teuthology.orchestra.run.smithi090.stdout:Downloading Packages: 2024-04-06T13:55:00.034 INFO:teuthology.orchestra.run.smithi105.stdout:(8/146): ceph-mon-18.2.2-1086.g052adcad.el8.x86 6.1 MB/s | 4.8 MB 00:00 2024-04-06T13:55:00.134 INFO:teuthology.orchestra.run.smithi105.stdout:(9/146): ceph-selinux-18.2.2-1086.g052adcad.el8 243 kB/s | 24 kB 00:00 2024-04-06T13:55:00.811 INFO:teuthology.orchestra.run.smithi090.stdout:(1/146): ceph-18.2.2-1086.g052adcad.el8.x86_64. 10 kB/s | 6.4 kB 00:00 2024-04-06T13:55:01.684 INFO:teuthology.orchestra.run.smithi105.stdout:(10/146): ceph-radosgw-18.2.2-1086.g052adcad.el 6.5 MB/s | 13 MB 00:01 2024-04-06T13:55:01.728 INFO:teuthology.orchestra.run.smithi090.stdout:(2/146): ceph-fuse-18.2.2-1086.g052adcad.el8.x8 938 kB/s | 860 kB 00:00 2024-04-06T13:55:01.818 INFO:teuthology.orchestra.run.smithi105.stdout:(11/146): cephfs-mirror-18.2.2-1086.g052adcad.e 1.9 MB/s | 261 kB 00:00 2024-04-06T13:55:01.829 INFO:teuthology.orchestra.run.smithi134.stdout:(17/146): librgw2-18.2.2-1086.g052adcad.el8.x86 2.8 MB/s | 6.5 MB 00:02 2024-04-06T13:55:01.887 INFO:teuthology.orchestra.run.smithi090.stdout:(3/146): ceph-immutable-object-cache-18.2.2-108 1.0 MB/s | 165 kB 00:00 2024-04-06T13:55:01.893 INFO:teuthology.orchestra.run.smithi105.stdout:(12/146): libcephfs-devel-18.2.2-1086.g052adcad 413 kB/s | 31 kB 00:00 2024-04-06T13:55:01.937 INFO:teuthology.orchestra.run.smithi134.stdout:(18/146): python3-ceph-argparse-18.2.2-1086.g05 412 kB/s | 45 kB 00:00 2024-04-06T13:55:02.152 INFO:teuthology.orchestra.run.smithi105.stdout:(13/146): libcephfs2-18.2.2-1086.g052adcad.el8. 3.0 MB/s | 781 kB 00:00 2024-04-06T13:55:02.235 INFO:teuthology.orchestra.run.smithi105.stdout:(14/146): libcephsqlite-18.2.2-1086.g052adcad.e 1.7 MB/s | 142 kB 00:00 2024-04-06T13:55:02.319 INFO:teuthology.orchestra.run.smithi105.stdout:(15/146): librados-devel-18.2.2-1086.g052adcad. 1.6 MB/s | 139 kB 00:00 2024-04-06T13:55:02.486 INFO:teuthology.orchestra.run.smithi105.stdout:(16/146): libradosstriper1-18.2.2-1086.g052adca 2.8 MB/s | 475 kB 00:00 2024-04-06T13:55:02.487 INFO:teuthology.orchestra.run.smithi134.stdout:(19/146): python3-ceph-common-18.2.2-1086.g052a 234 kB/s | 129 kB 00:00 2024-04-06T13:55:02.681 INFO:teuthology.orchestra.run.smithi134.stdout:(20/146): python3-cephfs-18.2.2-1086.g052adcad. 1.0 MB/s | 196 kB 00:00 2024-04-06T13:55:02.779 INFO:teuthology.orchestra.run.smithi090.stdout:(4/146): ceph-base-18.2.2-1086.g052adcad.el8.x8 2.1 MB/s | 5.5 MB 00:02 2024-04-06T13:55:02.988 INFO:teuthology.orchestra.run.smithi134.stdout:(21/146): python3-rados-18.2.2-1086.g052adcad.e 1.2 MB/s | 370 kB 00:00 2024-04-06T13:55:03.397 INFO:teuthology.orchestra.run.smithi134.stdout:(22/146): python3-rbd-18.2.2-1086.g052adcad.el8 893 kB/s | 365 kB 00:00 2024-04-06T13:55:03.454 INFO:teuthology.orchestra.run.smithi090.stdout:(5/146): ceph-mgr-18.2.2-1086.g052adcad.el8.x86 2.3 MB/s | 1.6 MB 00:00 2024-04-06T13:55:03.538 INFO:teuthology.orchestra.run.smithi090.stdout:(6/146): ceph-mds-18.2.2-1086.g052adcad.el8.x86 1.4 MB/s | 2.3 MB 00:01 2024-04-06T13:55:03.539 INFO:teuthology.orchestra.run.smithi134.stdout:(23/146): python3-rgw-18.2.2-1086.g052adcad.el8 753 kB/s | 107 kB 00:00 2024-04-06T13:55:03.714 INFO:teuthology.orchestra.run.smithi134.stdout:(24/146): rbd-fuse-18.2.2-1086.g052adcad.el8.x8 543 kB/s | 95 kB 00:00 2024-04-06T13:55:03.919 INFO:teuthology.orchestra.run.smithi105.stdout:(17/146): librgw2-18.2.2-1086.g052adcad.el8.x86 4.6 MB/s | 6.5 MB 00:01 2024-04-06T13:55:04.011 INFO:teuthology.orchestra.run.smithi105.stdout:(18/146): python3-ceph-argparse-18.2.2-1086.g05 485 kB/s | 45 kB 00:00 2024-04-06T13:55:04.103 INFO:teuthology.orchestra.run.smithi105.stdout:(19/146): python3-ceph-common-18.2.2-1086.g052a 1.4 MB/s | 129 kB 00:00 2024-04-06T13:55:04.195 INFO:teuthology.orchestra.run.smithi105.stdout:(20/146): python3-cephfs-18.2.2-1086.g052adcad. 2.1 MB/s | 196 kB 00:00 2024-04-06T13:55:04.354 INFO:teuthology.orchestra.run.smithi105.stdout:(21/146): python3-rados-18.2.2-1086.g052adcad.e 2.3 MB/s | 370 kB 00:00 2024-04-06T13:55:04.604 INFO:teuthology.orchestra.run.smithi105.stdout:(22/146): ceph-osd-18.2.2-1086.g052adcad.el8.x8 3.5 MB/s | 18 MB 00:05 2024-04-06T13:55:04.621 INFO:teuthology.orchestra.run.smithi105.stdout:(23/146): python3-rbd-18.2.2-1086.g052adcad.el8 1.3 MB/s | 365 kB 00:00 2024-04-06T13:55:04.680 INFO:teuthology.orchestra.run.smithi105.stdout:(24/146): python3-rgw-18.2.2-1086.g052adcad.el8 1.4 MB/s | 107 kB 00:00 2024-04-06T13:55:04.705 INFO:teuthology.orchestra.run.smithi105.stdout:(25/146): rbd-fuse-18.2.2-1086.g052adcad.el8.x8 1.1 MB/s | 95 kB 00:00 2024-04-06T13:55:04.772 INFO:teuthology.orchestra.run.smithi090.stdout:(7/146): ceph-mon-18.2.2-1086.g052adcad.el8.x86 3.7 MB/s | 4.8 MB 00:01 2024-04-06T13:55:04.772 INFO:teuthology.orchestra.run.smithi105.stdout:(26/146): rbd-nbd-18.2.2-1086.g052adcad.el8.x86 2.7 MB/s | 182 kB 00:00 2024-04-06T13:55:04.806 INFO:teuthology.orchestra.run.smithi134.stdout:(25/146): ceph-test-18.2.2-1086.g052adcad.el8.x 6.3 MB/s | 47 MB 00:07 2024-04-06T13:55:04.847 INFO:teuthology.orchestra.run.smithi105.stdout:(27/146): ceph-grafana-dashboards-18.2.2-1086.g 310 kB/s | 23 kB 00:00 2024-04-06T13:55:04.939 INFO:teuthology.orchestra.run.smithi105.stdout:(28/146): ceph-mgr-cephadm-18.2.2-1086.g052adca 1.5 MB/s | 138 kB 00:00 2024-04-06T13:55:04.965 INFO:teuthology.orchestra.run.smithi134.stdout:(26/146): rbd-nbd-18.2.2-1086.g052adcad.el8.x86 1.1 MB/s | 182 kB 00:00 2024-04-06T13:55:05.040 INFO:teuthology.orchestra.run.smithi134.stdout:(27/146): ceph-grafana-dashboards-18.2.2-1086.g 311 kB/s | 23 kB 00:00 2024-04-06T13:55:05.124 INFO:teuthology.orchestra.run.smithi134.stdout:(28/146): ceph-mgr-cephadm-18.2.2-1086.g052adca 1.6 MB/s | 138 kB 00:00 2024-04-06T13:55:05.531 INFO:teuthology.orchestra.run.smithi105.stdout:(29/146): rbd-mirror-18.2.2-1086.g052adcad.el8. 4.1 MB/s | 3.5 MB 00:00 2024-04-06T13:55:06.215 INFO:teuthology.orchestra.run.smithi105.stdout:(30/146): ceph-mgr-dashboard-18.2.2-1086.g052ad 2.8 MB/s | 3.5 MB 00:01 2024-04-06T13:55:06.298 INFO:teuthology.orchestra.run.smithi105.stdout:(31/146): ceph-mgr-modules-core-18.2.2-1086.g05 2.9 MB/s | 245 kB 00:00 2024-04-06T13:55:06.315 INFO:teuthology.orchestra.run.smithi134.stdout:(29/146): rbd-mirror-18.2.2-1086.g052adcad.el8. 1.3 MB/s | 3.5 MB 00:02 2024-04-06T13:55:06.366 INFO:teuthology.orchestra.run.smithi134.stdout:(30/146): ceph-mgr-dashboard-18.2.2-1086.g052ad 2.8 MB/s | 3.5 MB 00:01 2024-04-06T13:55:06.373 INFO:teuthology.orchestra.run.smithi105.stdout:(32/146): ceph-mgr-rook-18.2.2-1086.g052adcad.e 657 kB/s | 49 kB 00:00 2024-04-06T13:55:06.465 INFO:teuthology.orchestra.run.smithi105.stdout:(33/146): ceph-prometheus-alerts-18.2.2-1086.g0 161 kB/s | 15 kB 00:00 2024-04-06T13:55:06.508 INFO:teuthology.orchestra.run.smithi134.stdout:(31/146): ceph-mgr-modules-core-18.2.2-1086.g05 1.7 MB/s | 245 kB 00:00 2024-04-06T13:55:06.557 INFO:teuthology.orchestra.run.smithi105.stdout:(34/146): ceph-volume-18.2.2-1086.g052adcad.el8 2.8 MB/s | 263 kB 00:00 2024-04-06T13:55:06.600 INFO:teuthology.orchestra.run.smithi134.stdout:(32/146): ceph-mgr-rook-18.2.2-1086.g052adcad.e 538 kB/s | 49 kB 00:00 2024-04-06T13:55:06.641 INFO:teuthology.orchestra.run.smithi105.stdout:(35/146): cephadm-18.2.2-1086.g052adcad.el8.noa 2.0 MB/s | 173 kB 00:00 2024-04-06T13:55:06.691 INFO:teuthology.orchestra.run.smithi134.stdout:(33/146): ceph-prometheus-alerts-18.2.2-1086.g0 161 kB/s | 15 kB 00:00 2024-04-06T13:55:06.742 INFO:teuthology.orchestra.run.smithi105.stdout:(36/146): cephfs-top-18.2.2-1086.g052adcad.el8. 181 kB/s | 18 kB 00:00 2024-04-06T13:55:06.842 INFO:teuthology.orchestra.run.smithi134.stdout:(34/146): ceph-volume-18.2.2-1086.g052adcad.el8 1.7 MB/s | 263 kB 00:00 2024-04-06T13:55:06.917 INFO:teuthology.orchestra.run.smithi134.stdout:(35/146): cephadm-18.2.2-1086.g052adcad.el8.noa 2.3 MB/s | 173 kB 00:00 2024-04-06T13:55:07.018 INFO:teuthology.orchestra.run.smithi134.stdout:(36/146): cephfs-top-18.2.2-1086.g052adcad.el8. 181 kB/s | 18 kB 00:00 2024-04-06T13:55:07.175 INFO:teuthology.orchestra.run.smithi105.stdout:(37/146): gperftools-libs-2.7-9.el8.x86_64.rpm 707 kB/s | 306 kB 00:00 2024-04-06T13:55:07.275 INFO:teuthology.orchestra.run.smithi105.stdout:(38/146): liboath-2.6.2-3.el8.x86_64.rpm 589 kB/s | 59 kB 00:00 2024-04-06T13:55:07.389 INFO:teuthology.orchestra.run.smithi090.stdout:(8/146): ceph-common-18.2.2-1086.g052adcad.el8. 3.0 MB/s | 22 MB 00:07 2024-04-06T13:55:07.392 INFO:teuthology.orchestra.run.smithi105.stdout:(39/146): ceph-mgr-diskprediction-local-18.2.2- 4.0 MB/s | 7.4 MB 00:01 2024-04-06T13:55:07.417 INFO:teuthology.orchestra.run.smithi105.stdout:(40/146): libunwind-1.3.1-3.el8.x86_64.rpm 529 kB/s | 75 kB 00:00 2024-04-06T13:55:07.459 INFO:teuthology.orchestra.run.smithi134.stdout:(37/146): gperftools-libs-2.7-9.el8.x86_64.rpm 693 kB/s | 306 kB 00:00 2024-04-06T13:55:07.472 INFO:teuthology.orchestra.run.smithi090.stdout:(9/146): ceph-selinux-18.2.2-1086.g052adcad.el8 292 kB/s | 24 kB 00:00 2024-04-06T13:55:07.501 INFO:teuthology.orchestra.run.smithi105.stdout:(41/146): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 524 kB/s | 44 kB 00:00 2024-04-06T13:55:07.585 INFO:teuthology.orchestra.run.smithi134.stdout:(38/146): liboath-2.6.2-3.el8.x86_64.rpm 472 kB/s | 59 kB 00:00 2024-04-06T13:55:07.627 INFO:teuthology.orchestra.run.smithi105.stdout:(42/146): python3-beautifulsoup4-4.6.3-2.el8.1. 1.4 MB/s | 185 kB 00:00 2024-04-06T13:55:07.693 INFO:teuthology.orchestra.run.smithi134.stdout:(39/146): libunwind-1.3.1-3.el8.x86_64.rpm 693 kB/s | 75 kB 00:00 2024-04-06T13:55:07.710 INFO:teuthology.orchestra.run.smithi105.stdout:(43/146): python3-cachetools-3.1.1-4.el8.noarch 397 kB/s | 33 kB 00:00 2024-04-06T13:55:07.785 INFO:teuthology.orchestra.run.smithi105.stdout:(44/146): python3-certifi-2018.10.15-7.el8.noar 217 kB/s | 16 kB 00:00 2024-04-06T13:55:07.827 INFO:teuthology.orchestra.run.smithi134.stdout:(40/146): python3-asyncssh-2.7.0-2.el8.noarch.r 3.5 MB/s | 476 kB 00:00 2024-04-06T13:55:07.869 INFO:teuthology.orchestra.run.smithi105.stdout:(45/146): python3-asyncssh-2.7.0-2.el8.noarch.r 999 kB/s | 476 kB 00:00 2024-04-06T13:55:07.894 INFO:teuthology.orchestra.run.smithi134.stdout:(41/146): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 654 kB/s | 44 kB 00:00 2024-04-06T13:55:07.936 INFO:teuthology.orchestra.run.smithi105.stdout:(46/146): python3-cheroot-8.5.2-1.el8.noarch.rp 1.1 MB/s | 173 kB 00:00 2024-04-06T13:55:07.978 INFO:teuthology.orchestra.run.smithi134.stdout:(42/146): python3-beautifulsoup4-4.6.3-2.el8.1. 2.2 MB/s | 185 kB 00:00 2024-04-06T13:55:08.011 INFO:teuthology.orchestra.run.smithi105.stdout:(47/146): python3-cherrypy-18.4.0-1.el8.noarch. 2.7 MB/s | 384 kB 00:00 2024-04-06T13:55:08.037 INFO:teuthology.orchestra.run.smithi105.stdout:(48/146): python3-google-auth-1.1.1-10.el8.noar 810 kB/s | 81 kB 00:00 2024-04-06T13:55:08.044 INFO:teuthology.orchestra.run.smithi134.stdout:(43/146): python3-cachetools-3.1.1-4.el8.noarch 495 kB/s | 33 kB 00:00 2024-04-06T13:55:08.087 INFO:teuthology.orchestra.run.smithi105.stdout:(49/146): python3-jaraco-6.2-6.el8.noarch.rpm 147 kB/s | 11 kB 00:00 2024-04-06T13:55:08.111 INFO:teuthology.orchestra.run.smithi134.stdout:(44/146): python3-certifi-2018.10.15-7.el8.noar 245 kB/s | 16 kB 00:00 2024-04-06T13:55:08.120 INFO:teuthology.orchestra.run.smithi105.stdout:(50/146): python3-jaraco-functools-2.0-4.el8.no 210 kB/s | 18 kB 00:00 2024-04-06T13:55:08.195 INFO:teuthology.orchestra.run.smithi134.stdout:(45/146): python3-cheroot-8.5.2-1.el8.noarch.rp 2.0 MB/s | 173 kB 00:00 2024-04-06T13:55:08.221 INFO:teuthology.orchestra.run.smithi105.stdout:(51/146): python3-logutils-0.3.5-11.el8.noarch. 487 kB/s | 49 kB 00:00 2024-04-06T13:55:08.280 INFO:teuthology.orchestra.run.smithi105.stdout:(52/146): python3-kubernetes-11.0.0-6.el8.noarc 6.1 MB/s | 1.2 MB 00:00 2024-04-06T13:55:08.287 INFO:teuthology.orchestra.run.smithi134.stdout:(46/146): python3-cherrypy-18.4.0-1.el8.noarch. 4.1 MB/s | 384 kB 00:00 2024-04-06T13:55:08.296 INFO:teuthology.orchestra.run.smithi105.stdout:(53/146): python3-more-itertools-7.2.0-3.el8.no 778 kB/s | 59 kB 00:00 2024-04-06T13:55:08.355 INFO:teuthology.orchestra.run.smithi105.stdout:(54/146): python3-natsort-7.1.1-2.el8.noarch.rp 801 kB/s | 60 kB 00:00 2024-04-06T13:55:08.388 INFO:teuthology.orchestra.run.smithi134.stdout:(47/146): python3-google-auth-1.1.1-10.el8.noar 812 kB/s | 81 kB 00:00 2024-04-06T13:55:08.414 INFO:teuthology.orchestra.run.smithi105.stdout:(55/146): python3-pecan-1.3.2-9.el8.noarch.rpm 2.4 MB/s | 283 kB 00:00 2024-04-06T13:55:08.431 INFO:teuthology.orchestra.run.smithi105.stdout:(56/146): python3-portend-2.6-1.el8.noarch.rpm 207 kB/s | 16 kB 00:00 2024-04-06T13:55:08.463 INFO:teuthology.orchestra.run.smithi134.stdout:(48/146): python3-jaraco-6.2-6.el8.noarch.rpm 149 kB/s | 11 kB 00:00 2024-04-06T13:55:08.523 INFO:teuthology.orchestra.run.smithi105.stdout:(57/146): python3-repoze-lru-0.7-6.el8.noarch.r 310 kB/s | 34 kB 00:00 2024-04-06T13:55:08.538 INFO:teuthology.orchestra.run.smithi134.stdout:(49/146): python3-jaraco-functools-2.0-4.el8.no 234 kB/s | 18 kB 00:00 2024-04-06T13:55:08.548 INFO:teuthology.orchestra.run.smithi105.stdout:(58/146): python3-routes-2.4.1-12.el8.noarch.rp 1.6 MB/s | 196 kB 00:00 2024-04-06T13:55:08.607 INFO:teuthology.orchestra.run.smithi105.stdout:(59/146): python3-rsa-4.9-2.el8.noarch.rpm 769 kB/s | 64 kB 00:00 2024-04-06T13:55:08.632 INFO:teuthology.orchestra.run.smithi105.stdout:(60/146): python3-simplegeneric-0.8.1-17.el8.no 224 kB/s | 19 kB 00:00 2024-04-06T13:55:08.690 INFO:teuthology.orchestra.run.smithi105.stdout:(61/146): python3-singledispatch-3.4.0.3-18.el8 287 kB/s | 24 kB 00:00 2024-04-06T13:55:08.710 INFO:teuthology.orchestra.run.smithi134.stdout:(50/146): python3-kubernetes-11.0.0-6.el8.noarc 6.8 MB/s | 1.2 MB 00:00 2024-04-06T13:55:08.716 INFO:teuthology.orchestra.run.smithi105.stdout:(62/146): python3-tempora-1.14.1-5.el8.noarch.r 350 kB/s | 29 kB 00:00 2024-04-06T13:55:08.772 INFO:teuthology.orchestra.run.smithi090.stdout:(10/146): ceph-radosgw-18.2.2-1086.g052adcad.el 3.3 MB/s | 13 MB 00:04 2024-04-06T13:55:08.797 INFO:teuthology.orchestra.run.smithi134.stdout:(51/146): python3-logutils-0.3.5-11.el8.noarch. 559 kB/s | 49 kB 00:00 2024-04-06T13:55:08.806 INFO:teuthology.orchestra.run.smithi105.stdout:(63/146): python3-trustme-0.6.0-4.el8.noarch.rp 239 kB/s | 27 kB 00:00 2024-04-06T13:55:08.842 INFO:teuthology.orchestra.run.smithi105.stdout:(64/146): python3-waitress-1.4.3-1.el8.noarch.r 2.0 MB/s | 255 kB 00:00 2024-04-06T13:55:08.881 INFO:teuthology.orchestra.run.smithi090.stdout:(11/146): cephfs-mirror-18.2.2-1086.g052adcad.e 2.4 MB/s | 261 kB 00:00 2024-04-06T13:55:08.889 INFO:teuthology.orchestra.run.smithi134.stdout:(52/146): python3-more-itertools-7.2.0-3.el8.no 641 kB/s | 59 kB 00:00 2024-04-06T13:55:08.908 INFO:teuthology.orchestra.run.smithi105.stdout:(65/146): python3-webob-1.8.5-1.el8.1.noarch.rp 2.4 MB/s | 251 kB 00:00 2024-04-06T13:55:08.950 INFO:teuthology.orchestra.run.smithi105.stdout:(66/146): python3-websocket-client-0.56.0-5.el8 565 kB/s | 61 kB 00:00 2024-04-06T13:55:08.981 INFO:teuthology.orchestra.run.smithi134.stdout:(53/146): python3-natsort-7.1.1-2.el8.noarch.rp 658 kB/s | 60 kB 00:00 2024-04-06T13:55:08.990 INFO:teuthology.orchestra.run.smithi090.stdout:(12/146): libcephfs-devel-18.2.2-1086.g052adcad 286 kB/s | 31 kB 00:00 2024-04-06T13:55:09.001 INFO:teuthology.orchestra.run.smithi105.stdout:(67/146): python3-webtest-2.0.33-1.el8.noarch.r 928 kB/s | 85 kB 00:00 2024-04-06T13:55:09.034 INFO:teuthology.orchestra.run.smithi105.stdout:(68/146): python3-zc-lockfile-2.0-2.el8.noarch. 274 kB/s | 23 kB 00:00 2024-04-06T13:55:09.059 INFO:teuthology.orchestra.run.smithi105.stdout:(69/146): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.7 MB/s | 70 kB 00:00 2024-04-06T13:55:09.081 INFO:teuthology.orchestra.run.smithi134.stdout:(54/146): python3-pecan-1.3.2-9.el8.noarch.rpm 2.8 MB/s | 283 kB 00:00 2024-04-06T13:55:09.174 INFO:teuthology.orchestra.run.smithi134.stdout:(55/146): python3-portend-2.6-1.el8.noarch.rpm 170 kB/s | 16 kB 00:00 2024-04-06T13:55:09.243 INFO:teuthology.orchestra.run.smithi105.stdout:(70/146): thrift-0.13.0-2.el8.x86_64.rpm 7.2 MB/s | 1.7 MB 00:00 2024-04-06T13:55:09.274 INFO:teuthology.orchestra.run.smithi134.stdout:(56/146): python3-repoze-lru-0.7-6.el8.noarch.r 337 kB/s | 34 kB 00:00 2024-04-06T13:55:09.307 INFO:teuthology.orchestra.run.smithi090.stdout:(13/146): libcephfs2-18.2.2-1086.g052adcad.el8. 2.4 MB/s | 781 kB 00:00 2024-04-06T13:55:09.335 INFO:teuthology.orchestra.run.smithi105.stdout:(71/146): python3-jsonpointer-1.10-11.el8.noarc 73 kB/s | 20 kB 00:00 2024-04-06T13:55:09.416 INFO:teuthology.orchestra.run.smithi134.stdout:(57/146): python3-routes-2.4.1-12.el8.noarch.rp 1.4 MB/s | 196 kB 00:00 2024-04-06T13:55:09.448 INFO:teuthology.orchestra.run.smithi090.stdout:(14/146): libcephsqlite-18.2.2-1086.g052adcad.e 1.0 MB/s | 142 kB 00:00 2024-04-06T13:55:09.550 INFO:teuthology.orchestra.run.smithi134.stdout:(58/146): ceph-mgr-diskprediction-local-18.2.2- 2.3 MB/s | 7.4 MB 00:03 2024-04-06T13:55:09.567 INFO:teuthology.orchestra.run.smithi090.stdout:(15/146): librados-devel-18.2.2-1086.g052adcad. 1.1 MB/s | 139 kB 00:00 2024-04-06T13:55:09.575 INFO:teuthology.orchestra.run.smithi134.stdout:(59/146): python3-rsa-4.9-2.el8.noarch.rpm 404 kB/s | 64 kB 00:00 2024-04-06T13:55:09.667 INFO:teuthology.orchestra.run.smithi134.stdout:(60/146): python3-singledispatch-3.4.0.3-18.el8 261 kB/s | 24 kB 00:00 2024-04-06T13:55:09.717 INFO:teuthology.orchestra.run.smithi090.stdout:(16/146): libradosstriper1-18.2.2-1086.g052adca 3.1 MB/s | 475 kB 00:00 2024-04-06T13:55:09.735 INFO:teuthology.orchestra.run.smithi105.stdout:(72/146): python3-pyasn1-modules-0.3.7-6.el8.no 275 kB/s | 110 kB 00:00 2024-04-06T13:55:09.770 INFO:teuthology.orchestra.run.smithi134.stdout:(61/146): python3-tempora-1.14.1-5.el8.noarch.r 283 kB/s | 29 kB 00:00 2024-04-06T13:55:09.791 INFO:teuthology.orchestra.run.smithi105.stdout:(73/146): python3-prettytable-0.7.2-14.el8.noar 80 kB/s | 44 kB 00:00 2024-04-06T13:55:09.834 INFO:teuthology.orchestra.run.smithi134.stdout:(62/146): python3-simplegeneric-0.8.1-17.el8.no 66 kB/s | 19 kB 00:00 2024-04-06T13:55:09.859 INFO:teuthology.orchestra.run.smithi134.stdout:(63/146): python3-trustme-0.6.0-4.el8.noarch.rp 311 kB/s | 27 kB 00:00 2024-04-06T13:55:09.943 INFO:teuthology.orchestra.run.smithi134.stdout:(64/146): python3-webob-1.8.5-1.el8.1.noarch.rp 2.9 MB/s | 251 kB 00:00 2024-04-06T13:55:10.043 INFO:teuthology.orchestra.run.smithi134.stdout:(65/146): python3-websocket-client-0.56.0-5.el8 613 kB/s | 61 kB 00:00 2024-04-06T13:55:10.094 INFO:teuthology.orchestra.run.smithi105.stdout:(74/146): python3-jsonpatch-1.21-2.el8.noarch.r 74 kB/s | 27 kB 00:00 2024-04-06T13:55:10.152 INFO:teuthology.orchestra.run.smithi134.stdout:(66/146): python3-webtest-2.0.33-1.el8.noarch.r 788 kB/s | 85 kB 00:00 2024-04-06T13:55:10.177 INFO:teuthology.orchestra.run.smithi134.stdout:(67/146): python3-waitress-1.4.3-1.el8.noarch.r 745 kB/s | 255 kB 00:00 2024-04-06T13:55:10.186 INFO:teuthology.orchestra.run.smithi105.stdout:(75/146): python3-pyasn1-0.3.7-6.el8.noarch.rpm 320 kB/s | 126 kB 00:00 2024-04-06T13:55:10.245 INFO:teuthology.orchestra.run.smithi105.stdout:(76/146): python3-werkzeug-0.12.2-4.el8.noarch. 3.0 MB/s | 457 kB 00:00 2024-04-06T13:55:10.369 INFO:teuthology.orchestra.run.smithi134.stdout:(68/146): ceph-common-18.2.2-1086.g052adcad.el8 1.4 MB/s | 22 MB 00:15 2024-04-06T13:55:10.394 INFO:teuthology.orchestra.run.smithi134.stdout:(69/146): python3-zc-lockfile-2.0-2.el8.noarch. 95 kB/s | 23 kB 00:00 2024-04-06T13:55:10.415 INFO:teuthology.orchestra.run.smithi105.stdout:(77/146): flex-2.6.1-9.el8.x86_64.rpm 1.8 MB/s | 320 kB 00:00 2024-04-06T13:55:10.420 INFO:teuthology.orchestra.run.smithi134.stdout:(70/146): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.4 MB/s | 70 kB 00:00 2024-04-06T13:55:10.462 INFO:teuthology.orchestra.run.smithi105.stdout:(78/146): bison-3.0.4-10.el8.x86_64.rpm 2.4 MB/s | 688 kB 00:00 2024-04-06T13:55:10.537 INFO:teuthology.orchestra.run.smithi134.stdout:(71/146): python3-prettytable-0.7.2-14.el8.noar 396 kB/s | 44 kB 00:00 2024-04-06T13:55:10.562 INFO:teuthology.orchestra.run.smithi134.stdout:(72/146): python3-jsonpointer-1.10-11.el8.noarc 119 kB/s | 20 kB 00:00 2024-04-06T13:55:10.604 INFO:teuthology.orchestra.run.smithi134.stdout:(73/146): thrift-0.13.0-2.el8.x86_64.rpm 4.1 MB/s | 1.7 MB 00:00 2024-04-06T13:55:10.638 INFO:teuthology.orchestra.run.smithi105.stdout:(79/146): libtool-2.4.6-25.el8.x86_64.rpm 3.1 MB/s | 709 kB 00:00 2024-04-06T13:55:10.643 INFO:teuthology.orchestra.run.smithi090.stdout:(17/146): ceph-osd-18.2.2-1086.g052adcad.el8.x8 2.6 MB/s | 18 MB 00:07 2024-04-06T13:55:10.679 INFO:teuthology.orchestra.run.smithi134.stdout:(74/146): python3-pyasn1-modules-0.3.7-6.el8.no 775 kB/s | 110 kB 00:00 2024-04-06T13:55:10.705 INFO:teuthology.orchestra.run.smithi134.stdout:(75/146): python3-jsonpatch-1.21-2.el8.noarch.r 187 kB/s | 27 kB 00:00 2024-04-06T13:55:10.738 INFO:teuthology.orchestra.run.smithi105.stdout:(80/146): python3-markupsafe-0.23-19.el8.x86_64 141 kB/s | 39 kB 00:00 2024-04-06T13:55:10.743 INFO:teuthology.orchestra.run.smithi090.stdout:(18/146): python3-ceph-argparse-18.2.2-1086.g05 448 kB/s | 45 kB 00:00 2024-04-06T13:55:10.755 INFO:teuthology.orchestra.run.smithi134.stdout:(76/146): python3-pyasn1-0.3.7-6.el8.noarch.rpm 839 kB/s | 126 kB 00:00 2024-04-06T13:55:10.822 INFO:teuthology.orchestra.run.smithi134.stdout:(77/146): python3-werkzeug-0.12.2-4.el8.noarch. 3.1 MB/s | 457 kB 00:00 2024-04-06T13:55:10.864 INFO:teuthology.orchestra.run.smithi134.stdout:(78/146): bison-3.0.4-10.el8.x86_64.rpm 4.2 MB/s | 688 kB 00:00 2024-04-06T13:55:10.889 INFO:teuthology.orchestra.run.smithi134.stdout:(79/146): flex-2.6.1-9.el8.x86_64.rpm 2.3 MB/s | 320 kB 00:00 2024-04-06T13:55:10.893 INFO:teuthology.orchestra.run.smithi090.stdout:(19/146): python3-ceph-common-18.2.2-1086.g052a 859 kB/s | 129 kB 00:00 2024-04-06T13:55:10.930 INFO:teuthology.orchestra.run.smithi105.stdout:(81/146): uuid-1.6.2-43.el8.x86_64.rpm 334 kB/s | 64 kB 00:00 2024-04-06T13:55:10.955 INFO:teuthology.orchestra.run.smithi105.stdout:(82/146): lttng-ust-2.8.1-11.el8.x86_64.rpm 817 kB/s | 259 kB 00:00 2024-04-06T13:55:11.060 INFO:teuthology.orchestra.run.smithi090.stdout:(20/146): python3-cephfs-18.2.2-1086.g052adcad. 1.2 MB/s | 196 kB 00:00 2024-04-06T13:55:11.173 INFO:teuthology.orchestra.run.smithi134.stdout:(80/146): libtool-2.4.6-25.el8.x86_64.rpm 2.0 MB/s | 709 kB 00:00 2024-04-06T13:55:11.231 INFO:teuthology.orchestra.run.smithi105.stdout:(83/146): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2024-04-06T13:55:11.231 INFO:teuthology.orchestra.run.smithi134.stdout:(81/146): python3-markupsafe-0.23-19.el8.x86_64 106 kB/s | 39 kB 00:00 2024-04-06T13:55:11.248 INFO:teuthology.orchestra.run.smithi105.stdout:(84/146): python3-pyOpenSSL-19.0.0-1.el8.noarch 326 kB/s | 103 kB 00:00 2024-04-06T13:55:11.257 INFO:teuthology.orchestra.run.smithi134.stdout:(82/146): lttng-ust-2.8.1-11.el8.x86_64.rpm 705 kB/s | 259 kB 00:00 2024-04-06T13:55:11.282 INFO:teuthology.orchestra.run.smithi134.stdout:(83/146): uuid-1.6.2-43.el8.x86_64.rpm 587 kB/s | 64 kB 00:00 2024-04-06T13:55:11.374 INFO:teuthology.orchestra.run.smithi134.stdout:(84/146): python3-pyOpenSSL-19.0.0-1.el8.noarch 728 kB/s | 103 kB 00:00 2024-04-06T13:55:11.583 INFO:teuthology.orchestra.run.smithi134.stdout:(85/146): python3-babel-2.5.1-7.el8.noarch.rpm 15 MB/s | 4.8 MB 00:00 2024-04-06T13:55:11.616 INFO:teuthology.orchestra.run.smithi134.stdout:(86/146): python3-jinja2-2.10.1-3.el8.noarch.rp 1.6 MB/s | 538 kB 00:00 2024-04-06T13:55:11.639 INFO:teuthology.orchestra.run.smithi105.stdout:(85/146): python3-numpy-1.14.3-10.el8.x86_64.rp 9.5 MB/s | 3.7 MB 00:00 2024-04-06T13:55:11.644 INFO:teuthology.orchestra.run.smithi090.stdout:(21/146): python3-rados-18.2.2-1086.g052adcad.e 634 kB/s | 370 kB 00:00 2024-04-06T13:55:11.673 INFO:teuthology.orchestra.run.smithi105.stdout:(86/146): python3-jinja2-2.10.1-3.el8.noarch.rp 1.2 MB/s | 538 kB 00:00 2024-04-06T13:55:11.783 INFO:teuthology.orchestra.run.smithi134.stdout:(87/146): python3-numpy-1.14.3-10.el8.x86_64.rp 9.1 MB/s | 3.7 MB 00:00 2024-04-06T13:55:11.946 INFO:teuthology.orchestra.run.smithi090.stdout:(22/146): python3-rbd-18.2.2-1086.g052adcad.el8 1.2 MB/s | 365 kB 00:00 2024-04-06T13:55:11.984 INFO:teuthology.orchestra.run.smithi134.stdout:(88/146): socat-1.7.4.1-1.el8.x86_64.rpm 807 kB/s | 323 kB 00:00 2024-04-06T13:55:12.034 INFO:teuthology.orchestra.run.smithi134.stdout:(89/146): python3-numpy-f2py-1.14.3-10.el8.x86_ 540 kB/s | 225 kB 00:00 2024-04-06T13:55:12.159 INFO:teuthology.orchestra.run.smithi134.stdout:(90/146): librdkafka-0.11.4-3.el8.x86_64.rpm 942 kB/s | 354 kB 00:00 2024-04-06T13:55:12.265 INFO:teuthology.orchestra.run.smithi105.stdout:(87/146): ceph-test-18.2.2-1086.g052adcad.el8.x 3.9 MB/s | 47 MB 00:12 2024-04-06T13:55:12.299 INFO:teuthology.orchestra.run.smithi105.stdout:(88/146): python3-numpy-f2py-1.14.3-10.el8.x86_ 360 kB/s | 225 kB 00:00 2024-04-06T13:55:12.301 INFO:teuthology.orchestra.run.smithi134.stdout:(91/146): openblas-threads-0.3.15-6.el8.x86_64. 16 MB/s | 5.2 MB 00:00 2024-04-06T13:55:12.313 INFO:teuthology.orchestra.run.smithi090.stdout:(23/146): librgw2-18.2.2-1086.g052adcad.el8.x86 2.5 MB/s | 6.5 MB 00:02 2024-04-06T13:55:12.326 INFO:teuthology.orchestra.run.smithi134.stdout:(92/146): python3-mako-1.0.6-14.el8.noarch.rpm 537 kB/s | 157 kB 00:00 2024-04-06T13:55:12.332 INFO:teuthology.orchestra.run.smithi105.stdout:(89/146): socat-1.7.4.1-1.el8.x86_64.rpm 467 kB/s | 323 kB 00:00 2024-04-06T13:55:12.370 INFO:teuthology.orchestra.run.smithi090.stdout:(24/146): python3-rgw-18.2.2-1086.g052adcad.el8 252 kB/s | 107 kB 00:00 2024-04-06T13:55:12.383 INFO:teuthology.orchestra.run.smithi105.stdout:(90/146): librdkafka-0.11.4-3.el8.x86_64.rpm 3.0 MB/s | 354 kB 00:00 2024-04-06T13:55:12.395 INFO:teuthology.orchestra.run.smithi090.stdout:(25/146): rbd-fuse-18.2.2-1086.g052adcad.el8.x8 1.1 MB/s | 95 kB 00:00 2024-04-06T13:55:12.435 INFO:teuthology.orchestra.run.smithi134.stdout:(93/146): openblas-0.3.15-6.el8.x86_64.rpm 18 MB/s | 5.0 MB 00:00 2024-04-06T13:55:12.460 INFO:teuthology.orchestra.run.smithi134.stdout:(94/146): libpq-13.11-1.el8.x86_64.rpm 1.4 MB/s | 198 kB 00:00 2024-04-06T13:55:12.462 INFO:teuthology.orchestra.run.smithi090.stdout:(26/146): rbd-nbd-18.2.2-1086.g052adcad.el8.x86 2.7 MB/s | 182 kB 00:00 2024-04-06T13:55:12.502 INFO:teuthology.orchestra.run.smithi134.stdout:(95/146): automake-1.16.1-8.el8.noarch.rpm 3.5 MB/s | 714 kB 00:00 2024-04-06T13:55:12.521 INFO:teuthology.orchestra.run.smithi090.stdout:(27/146): ceph-grafana-dashboards-18.2.2-1086.g 398 kB/s | 23 kB 00:00 2024-04-06T13:55:12.575 INFO:teuthology.orchestra.run.smithi105.stdout:(91/146): openblas-threads-0.3.15-6.el8.x86_64. 19 MB/s | 5.2 MB 00:00 2024-04-06T13:55:12.588 INFO:teuthology.orchestra.run.smithi090.stdout:(28/146): ceph-mgr-cephadm-18.2.2-1086.g052adca 2.0 MB/s | 138 kB 00:00 2024-04-06T13:55:12.600 INFO:teuthology.orchestra.run.smithi105.stdout:(92/146): python3-mako-1.0.6-14.el8.noarch.rpm 588 kB/s | 157 kB 00:00 2024-04-06T13:55:12.733 INFO:teuthology.orchestra.run.smithi105.stdout:(93/146): openblas-0.3.15-6.el8.x86_64.rpm 14 MB/s | 5.0 MB 00:00 2024-04-06T13:55:12.875 INFO:teuthology.orchestra.run.smithi105.stdout:(94/146): automake-1.16.1-8.el8.noarch.rpm 2.3 MB/s | 714 kB 00:00 2024-04-06T13:55:12.894 INFO:teuthology.orchestra.run.smithi134.stdout:(96/146): python3-scipy-1.0.0-21.module+el8.9.0 35 MB/s | 14 MB 00:00 2024-04-06T13:55:12.900 INFO:teuthology.orchestra.run.smithi105.stdout:(95/146): libpq-13.11-1.el8.x86_64.rpm 658 kB/s | 198 kB 00:00 2024-04-06T13:55:12.919 INFO:teuthology.orchestra.run.smithi134.stdout:(97/146): python3-pytz-2017.2-11.el8.noarch.rpm 118 kB/s | 54 kB 00:00 2024-04-06T13:55:12.926 INFO:teuthology.orchestra.run.smithi105.stdout:(96/146): jq-1.6-7.el8.x86_64.rpm 1.0 MB/s | 202 kB 00:00 2024-04-06T13:55:12.977 INFO:teuthology.orchestra.run.smithi134.stdout:(98/146): jq-1.6-7.el8.x86_64.rpm 374 kB/s | 202 kB 00:00 2024-04-06T13:55:13.047 INFO:teuthology.orchestra.run.smithi090.stdout:(29/146): rbd-mirror-18.2.2-1086.g052adcad.el8. 5.1 MB/s | 3.5 MB 00:00 2024-04-06T13:55:13.059 INFO:teuthology.orchestra.run.smithi105.stdout:(97/146): python3-pytz-2017.2-11.el8.noarch.rpm 295 kB/s | 54 kB 00:00 2024-04-06T13:55:13.093 INFO:teuthology.orchestra.run.smithi105.stdout:(98/146): oniguruma-6.8.2-2.1.el8_9.x86_64.rpm 1.1 MB/s | 188 kB 00:00 2024-04-06T13:55:13.186 INFO:teuthology.orchestra.run.smithi134.stdout:(99/146): oniguruma-6.8.2-2.1.el8_9.x86_64.rpm 647 kB/s | 188 kB 00:00 2024-04-06T13:55:13.237 INFO:teuthology.orchestra.run.smithi134.stdout:(100/146): postgresql-10.23-4.module+el8.9.0+21 4.7 MB/s | 1.5 MB 00:00 2024-04-06T13:55:13.260 INFO:teuthology.orchestra.run.smithi105.stdout:(99/146): postgresql-contrib-10.23-4.module+el8 4.8 MB/s | 812 kB 00:00 2024-04-06T13:55:13.270 INFO:teuthology.orchestra.run.smithi134.stdout:(101/146): postgresql-contrib-10.23-4.module+el 2.7 MB/s | 812 kB 00:00 2024-04-06T13:55:13.302 INFO:teuthology.orchestra.run.smithi105.stdout:(100/146): postgresql-10.23-4.module+el8.9.0+21 6.2 MB/s | 1.5 MB 00:00 2024-04-06T13:55:13.380 INFO:teuthology.orchestra.run.smithi090.stdout:(30/146): ceph-mgr-dashboard-18.2.2-1086.g052ad 4.5 MB/s | 3.5 MB 00:00 2024-04-06T13:55:13.404 INFO:teuthology.orchestra.run.smithi134.stdout:(102/146): libverto-devel-0.3.0-5.el8.x86_64.rp 134 kB/s | 18 kB 00:00 2024-04-06T13:55:13.421 INFO:teuthology.orchestra.run.smithi134.stdout:(103/146): python3-packaging-16.8-9.el8.noarch. 262 kB/s | 48 kB 00:00 2024-04-06T13:55:13.481 INFO:teuthology.orchestra.run.smithi090.stdout:(31/146): ceph-mgr-modules-core-18.2.2-1086.g05 2.4 MB/s | 245 kB 00:00 2024-04-06T13:55:13.488 INFO:teuthology.orchestra.run.smithi134.stdout:(104/146): python3-oauthlib-2.1.0-1.el8.noarch. 2.3 MB/s | 155 kB 00:00 2024-04-06T13:55:13.519 INFO:teuthology.orchestra.run.smithi105.stdout:(101/146): python3-scipy-1.0.0-21.module+el8.9. 22 MB/s | 14 MB 00:00 2024-04-06T13:55:13.544 INFO:teuthology.orchestra.run.smithi105.stdout:(102/146): libverto-devel-0.3.0-5.el8.x86_64.rp 74 kB/s | 18 kB 00:00 2024-04-06T13:55:13.548 INFO:teuthology.orchestra.run.smithi090.stdout:(32/146): ceph-mgr-rook-18.2.2-1086.g052adcad.e 737 kB/s | 49 kB 00:00 2024-04-06T13:55:13.603 INFO:teuthology.orchestra.run.smithi105.stdout:(103/146): python3-jwt-1.6.1-2.el8.noarch.rpm 511 kB/s | 43 kB 00:00 2024-04-06T13:55:13.613 INFO:teuthology.orchestra.run.smithi134.stdout:(105/146): postgresql-server-10.23-4.module+el8 12 MB/s | 5.1 MB 00:00 2024-04-06T13:55:13.615 INFO:teuthology.orchestra.run.smithi090.stdout:(33/146): ceph-prometheus-alerts-18.2.2-1086.g0 221 kB/s | 15 kB 00:00 2024-04-06T13:55:13.630 INFO:teuthology.orchestra.run.smithi134.stdout:(106/146): python3-jwt-1.6.1-2.el8.noarch.rpm 188 kB/s | 43 kB 00:00 2024-04-06T13:55:13.678 INFO:teuthology.orchestra.run.smithi105.stdout:(104/146): python3-oauthlib-2.1.0-1.el8.noarch. 1.1 MB/s | 155 kB 00:00 2024-04-06T13:55:13.681 INFO:teuthology.orchestra.run.smithi134.stdout:(107/146): python3-pycparser-2.14-14.el8.noarch 566 kB/s | 109 kB 00:00 2024-04-06T13:55:13.706 INFO:teuthology.orchestra.run.smithi134.stdout:(108/146): libaio-devel-0.3.112-1.el8.x86_64.rp 249 kB/s | 19 kB 00:00 2024-04-06T13:55:13.723 INFO:teuthology.orchestra.run.smithi090.stdout:(34/146): ceph-volume-18.2.2-1086.g052adcad.el8 2.4 MB/s | 263 kB 00:00 2024-04-06T13:55:13.731 INFO:teuthology.orchestra.run.smithi134.stdout:(109/146): python3-requests-oauthlib-1.0.0-1.el 363 kB/s | 43 kB 00:00 2024-04-06T13:55:13.745 INFO:teuthology.orchestra.run.smithi105.stdout:(105/146): postgresql-server-10.23-4.module+el8 10 MB/s | 5.1 MB 00:00 2024-04-06T13:55:13.773 INFO:teuthology.orchestra.run.smithi134.stdout:(110/146): libzstd-devel-1.4.4-1.el8.x86_64.rpm 476 kB/s | 44 kB 00:00 2024-04-06T13:55:13.799 INFO:teuthology.orchestra.run.smithi090.stdout:(35/146): cephadm-18.2.2-1086.g052adcad.el8.noa 2.2 MB/s | 173 kB 00:00 2024-04-06T13:55:13.806 INFO:teuthology.orchestra.run.smithi105.stdout:(106/146): python3-pycparser-2.14-14.el8.noarch 536 kB/s | 109 kB 00:00 2024-04-06T13:55:13.832 INFO:teuthology.orchestra.run.smithi134.stdout:(111/146): pcre2-utf16-10.32-2.el8.x86_64.rpm 2.2 MB/s | 229 kB 00:00 2024-04-06T13:55:13.837 INFO:teuthology.orchestra.run.smithi105.stdout:(107/146): python3-requests-oauthlib-1.0.0-1.el 268 kB/s | 43 kB 00:00 2024-04-06T13:55:13.866 INFO:teuthology.orchestra.run.smithi090.stdout:(36/146): cephfs-top-18.2.2-1086.g052adcad.el8. 271 kB/s | 18 kB 00:00 2024-04-06T13:55:13.871 INFO:teuthology.orchestra.run.smithi105.stdout:(108/146): libaio-devel-0.3.112-1.el8.x86_64.rp 149 kB/s | 19 kB 00:00 2024-04-06T13:55:13.907 INFO:teuthology.orchestra.run.smithi134.stdout:(112/146): pcre2-devel-10.32-2.el8.x86_64.rpm 2.9 MB/s | 605 kB 00:00 2024-04-06T13:55:13.930 INFO:teuthology.orchestra.run.smithi105.stdout:(109/146): pcre2-utf16-10.32-2.el8.x86_64.rpm 3.8 MB/s | 229 kB 00:00 2024-04-06T13:55:13.932 INFO:teuthology.orchestra.run.smithi134.stdout:(113/146): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.4 MB/s | 220 kB 00:00 2024-04-06T13:55:13.957 INFO:teuthology.orchestra.run.smithi134.stdout:(114/146): libselinux-devel-2.9-5.el8.x86_64.rp 1.6 MB/s | 200 kB 00:00 2024-04-06T13:55:13.972 INFO:teuthology.orchestra.run.smithi105.stdout:(110/146): pcre2-devel-10.32-2.el8.x86_64.rpm 4.4 MB/s | 605 kB 00:00 2024-04-06T13:55:13.997 INFO:teuthology.orchestra.run.smithi105.stdout:(111/146): libzstd-devel-1.4.4-1.el8.x86_64.rpm 231 kB/s | 44 kB 00:00 2024-04-06T13:55:14.066 INFO:teuthology.orchestra.run.smithi134.stdout:(115/146): keyutils-libs-devel-1.5.10-9.el8.x86 303 kB/s | 48 kB 00:00 2024-04-06T13:55:14.091 INFO:teuthology.orchestra.run.smithi134.stdout:(116/146): libsepol-devel-2.9-3.el8.x86_64.rpm 548 kB/s | 87 kB 00:00 2024-04-06T13:55:14.102 INFO:teuthology.orchestra.run.smithi105.stdout:(112/146): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.3 MB/s | 220 kB 00:00 2024-04-06T13:55:14.117 INFO:teuthology.orchestra.run.smithi134.stdout:(117/146): zlib-devel-1.2.11-18.el8_5.x86_64.rp 363 kB/s | 58 kB 00:00 2024-04-06T13:55:14.156 INFO:teuthology.orchestra.run.smithi105.stdout:(113/146): keyutils-libs-devel-1.5.10-9.el8.x86 303 kB/s | 48 kB 00:00 2024-04-06T13:55:14.159 INFO:teuthology.orchestra.run.smithi134.stdout:(118/146): libcom_err-devel-1.45.6-4.el8.x86_64 418 kB/s | 39 kB 00:00 2024-04-06T13:55:14.175 INFO:teuthology.orchestra.run.smithi090.stdout:(37/146): gperftools-libs-2.7-9.el8.x86_64.rpm 993 kB/s | 306 kB 00:00 2024-04-06T13:55:14.198 INFO:teuthology.orchestra.run.smithi105.stdout:(114/146): libsepol-devel-2.9-3.el8.x86_64.rpm 914 kB/s | 87 kB 00:00 2024-04-06T13:55:14.223 INFO:teuthology.orchestra.run.smithi105.stdout:(115/146): libselinux-devel-2.9-5.el8.x86_64.rp 796 kB/s | 200 kB 00:00 2024-04-06T13:55:14.234 INFO:teuthology.orchestra.run.smithi090.stdout:(38/146): liboath-2.6.2-3.el8.x86_64.rpm 993 kB/s | 59 kB 00:00 2024-04-06T13:55:14.267 INFO:teuthology.orchestra.run.smithi134.stdout:(119/146): fuse-2.9.7-17.el8.x86_64.rpm 473 kB/s | 83 kB 00:00 2024-04-06T13:55:14.290 INFO:teuthology.orchestra.run.smithi105.stdout:(116/146): libcom_err-devel-1.45.6-4.el8.x86_64 420 kB/s | 39 kB 00:00 2024-04-06T13:55:14.293 INFO:teuthology.orchestra.run.smithi134.stdout:(120/146): libquadmath-8.5.0-20.el8.x86_64.rpm 1.3 MB/s | 172 kB 00:00 2024-04-06T13:55:14.340 INFO:teuthology.orchestra.run.smithi105.stdout:(117/146): zlib-devel-1.2.11-18.el8_5.x86_64.rp 313 kB/s | 58 kB 00:00 2024-04-06T13:55:14.343 INFO:teuthology.orchestra.run.smithi090.stdout:(39/146): ceph-mgr-diskprediction-local-18.2.2- 5.7 MB/s | 7.4 MB 00:01 2024-04-06T13:55:14.368 INFO:teuthology.orchestra.run.smithi090.stdout:(40/146): libunwind-1.3.1-3.el8.x86_64.rpm 561 kB/s | 75 kB 00:00 2024-04-06T13:55:14.376 INFO:teuthology.orchestra.run.smithi134.stdout:(121/146): libgfortran-8.5.0-20.el8.x86_64.rpm 2.4 MB/s | 645 kB 00:00 2024-04-06T13:55:14.426 INFO:teuthology.orchestra.run.smithi134.stdout:(122/146): libuuid-devel-2.32.1-43.el8.x86_64.r 628 kB/s | 100 kB 00:00 2024-04-06T13:55:14.427 INFO:teuthology.orchestra.run.smithi090.stdout:(41/146): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 747 kB/s | 44 kB 00:00 2024-04-06T13:55:14.432 INFO:teuthology.orchestra.run.smithi105.stdout:(118/146): libgfortran-8.5.0-20.el8.x86_64.rpm 4.4 MB/s | 645 kB 00:00 2024-04-06T13:55:14.457 INFO:teuthology.orchestra.run.smithi105.stdout:(119/146): fuse-2.9.7-17.el8.x86_64.rpm 355 kB/s | 83 kB 00:00 2024-04-06T13:55:14.477 INFO:teuthology.orchestra.run.smithi134.stdout:(123/146): python3-cffi-1.11.5-6.el8.x86_64.rpm 1.3 MB/s | 238 kB 00:00 2024-04-06T13:55:14.494 INFO:teuthology.orchestra.run.smithi090.stdout:(42/146): python3-beautifulsoup4-4.6.3-2.el8.1. 2.7 MB/s | 185 kB 00:00 2024-04-06T13:55:14.533 INFO:teuthology.orchestra.run.smithi105.stdout:(120/146): libquadmath-8.5.0-20.el8.x86_64.rpm 897 kB/s | 172 kB 00:00 2024-04-06T13:55:14.560 INFO:teuthology.orchestra.run.smithi134.stdout:(124/146): xfsprogs-devel-5.0.0-12.el8.x86_64.r 395 kB/s | 73 kB 00:00 2024-04-06T13:55:14.586 INFO:teuthology.orchestra.run.smithi090.stdout:(43/146): python3-cachetools-3.1.1-4.el8.noarch 360 kB/s | 33 kB 00:00 2024-04-06T13:55:14.591 INFO:teuthology.orchestra.run.smithi105.stdout:(121/146): python3-cffi-1.11.5-6.el8.x86_64.rpm 1.7 MB/s | 238 kB 00:00 2024-04-06T13:55:14.602 INFO:teuthology.orchestra.run.smithi134.stdout:(125/146): elfutils-libelf-devel-0.189-3.el8.x8 493 kB/s | 62 kB 00:00 2024-04-06T13:55:14.619 INFO:teuthology.orchestra.run.smithi134.stdout:(126/146): NetworkManager-initscripts-updown-1. 748 kB/s | 144 kB 00:00 2024-04-06T13:55:14.645 INFO:teuthology.orchestra.run.smithi090.stdout:(44/146): python3-certifi-2018.10.15-7.el8.noar 280 kB/s | 16 kB 00:00 2024-04-06T13:55:14.687 INFO:teuthology.orchestra.run.smithi090.stdout:(45/146): python3-asyncssh-2.7.0-2.el8.noarch.r 1.4 MB/s | 476 kB 00:00 2024-04-06T13:55:14.720 INFO:teuthology.orchestra.run.smithi090.stdout:(46/146): python3-cheroot-8.5.2-1.el8.noarch.rp 2.2 MB/s | 173 kB 00:00 2024-04-06T13:55:14.725 INFO:teuthology.orchestra.run.smithi105.stdout:(122/146): libuuid-devel-2.32.1-43.el8.x86_64.r 341 kB/s | 100 kB 00:00 2024-04-06T13:55:14.728 INFO:teuthology.orchestra.run.smithi134.stdout:(127/146): krb5-devel-1.18.2-26.el8_9.x86_64.rp 3.3 MB/s | 562 kB 00:00 2024-04-06T13:55:14.767 INFO:teuthology.orchestra.run.smithi105.stdout:(123/146): xfsprogs-devel-5.0.0-12.el8.x86_64.r 310 kB/s | 73 kB 00:00 2024-04-06T13:55:14.781 INFO:teuthology.orchestra.run.smithi090.stdout:(47/146): python3-cherrypy-18.4.0-1.el8.noarch. 4.1 MB/s | 384 kB 00:00 2024-04-06T13:55:14.804 INFO:teuthology.orchestra.run.smithi090.stdout:(48/146): python3-google-auth-1.1.1-10.el8.noar 972 kB/s | 81 kB 00:00 2024-04-06T13:55:14.817 INFO:teuthology.orchestra.run.smithi105.stdout:(124/146): NetworkManager-initscripts-updown-1. 639 kB/s | 144 kB 00:00 2024-04-06T13:55:14.834 INFO:teuthology.orchestra.run.smithi105.stdout:(125/146): elfutils-libelf-devel-0.189-3.el8.x8 567 kB/s | 62 kB 00:00 2024-04-06T13:55:14.838 INFO:teuthology.orchestra.run.smithi090.stdout:(49/146): python3-jaraco-6.2-6.el8.noarch.rpm 189 kB/s | 11 kB 00:00 2024-04-06T13:55:14.864 INFO:teuthology.orchestra.run.smithi090.stdout:(50/146): python3-jaraco-functools-2.0-4.el8.no 299 kB/s | 18 kB 00:00 2024-04-06T13:55:14.903 INFO:teuthology.orchestra.run.smithi134.stdout:(128/146): openssl-devel-1.1.1k-12.el8_9.x86_64 7.7 MB/s | 2.3 MB 00:00 2024-04-06T13:55:14.922 INFO:teuthology.orchestra.run.smithi090.stdout:(51/146): python3-logutils-0.3.5-11.el8.noarch. 839 kB/s | 49 kB 00:00 2024-04-06T13:55:14.926 INFO:teuthology.orchestra.run.smithi105.stdout:(126/146): krb5-devel-1.18.2-26.el8_9.x86_64.rp 3.5 MB/s | 562 kB 00:00 2024-04-06T13:55:14.928 INFO:teuthology.orchestra.run.smithi134.stdout:(129/146): librabbitmq-0.9.0-5.el8_9.x86_64.rpm 152 kB/s | 47 kB 00:00 2024-04-06T13:55:14.962 INFO:teuthology.orchestra.run.smithi134.stdout:(130/146): python3-cryptography-3.2.1-7.el8_9.x 2.3 MB/s | 559 kB 00:00 2024-04-06T13:55:14.981 INFO:teuthology.orchestra.run.smithi090.stdout:(52/146): python3-more-itertools-7.2.0-3.el8.no 1.0 MB/s | 59 kB 00:00 2024-04-06T13:55:15.023 INFO:teuthology.orchestra.run.smithi090.stdout:(53/146): python3-kubernetes-11.0.0-6.el8.noarc 6.3 MB/s | 1.2 MB 00:00 2024-04-06T13:55:15.048 INFO:teuthology.orchestra.run.smithi090.stdout:(54/146): python3-natsort-7.1.1-2.el8.noarch.rp 901 kB/s | 60 kB 00:00 2024-04-06T13:55:15.090 INFO:teuthology.orchestra.run.smithi090.stdout:(55/146): python3-pecan-1.3.2-9.el8.noarch.rpm 4.1 MB/s | 283 kB 00:00 2024-04-06T13:55:15.093 INFO:teuthology.orchestra.run.smithi105.stdout:(127/146): openssl-devel-1.1.1k-12.el8_9.x86_64 8.4 MB/s | 2.3 MB 00:00 2024-04-06T13:55:15.115 INFO:teuthology.orchestra.run.smithi090.stdout:(56/146): python3-portend-2.6-1.el8.noarch.rpm 233 kB/s | 16 kB 00:00 2024-04-06T13:55:15.118 INFO:teuthology.orchestra.run.smithi105.stdout:(128/146): librabbitmq-0.9.0-5.el8_9.x86_64.rpm 166 kB/s | 47 kB 00:00 2024-04-06T13:55:15.129 INFO:teuthology.orchestra.run.smithi134.stdout:(131/146): elfutils-libelf-0.189-3.el8.x86_64.r 1.4 MB/s | 232 kB 00:00 2024-04-06T13:55:15.149 INFO:teuthology.orchestra.run.smithi090.stdout:(57/146): python3-repoze-lru-0.7-6.el8.noarch.r 576 kB/s | 34 kB 00:00 2024-04-06T13:55:15.152 INFO:teuthology.orchestra.run.smithi105.stdout:(129/146): python3-cryptography-3.2.1-7.el8_9.x 2.4 MB/s | 559 kB 00:00 2024-04-06T13:55:15.174 INFO:teuthology.orchestra.run.smithi090.stdout:(58/146): python3-routes-2.4.1-12.el8.noarch.rp 3.3 MB/s | 196 kB 00:00 2024-04-06T13:55:15.177 INFO:teuthology.orchestra.run.smithi105.stdout:(130/146): python3-packaging-16.8-9.el8.noarch. 575 kB/s | 48 kB 00:00 2024-04-06T13:55:15.225 INFO:teuthology.orchestra.run.smithi090.stdout:(59/146): python3-rsa-4.9-2.el8.noarch.rpm 852 kB/s | 64 kB 00:00 2024-04-06T13:55:15.250 INFO:teuthology.orchestra.run.smithi090.stdout:(60/146): python3-simplegeneric-0.8.1-17.el8.no 248 kB/s | 19 kB 00:00 2024-04-06T13:55:15.261 INFO:teuthology.orchestra.run.smithi105.stdout:(131/146): elfutils-libelf-0.189-3.el8.x86_64.r 2.7 MB/s | 232 kB 00:00 2024-04-06T13:55:15.284 INFO:teuthology.orchestra.run.smithi090.stdout:(61/146): python3-singledispatch-3.4.0.3-18.el8 408 kB/s | 24 kB 00:00 2024-04-06T13:55:15.304 INFO:teuthology.orchestra.run.smithi134.stdout:(132/146): elfutils-libs-0.189-3.el8.x86_64.rpm 1.7 MB/s | 303 kB 00:00 2024-04-06T13:55:15.310 INFO:teuthology.orchestra.run.smithi090.stdout:(62/146): python3-tempora-1.14.1-5.el8.noarch.r 489 kB/s | 29 kB 00:00 2024-04-06T13:55:15.344 INFO:teuthology.orchestra.run.smithi090.stdout:(63/146): python3-trustme-0.6.0-4.el8.noarch.rp 461 kB/s | 27 kB 00:00 2024-04-06T13:55:15.378 INFO:teuthology.orchestra.run.smithi090.stdout:(64/146): python3-waitress-1.4.3-1.el8.noarch.r 3.7 MB/s | 255 kB 00:00 2024-04-06T13:55:15.403 INFO:teuthology.orchestra.run.smithi105.stdout:(132/146): elfutils-libs-0.189-3.el8.x86_64.rpm 2.1 MB/s | 303 kB 00:00 2024-04-06T13:55:15.411 INFO:teuthology.orchestra.run.smithi090.stdout:(65/146): python3-webob-1.8.5-1.el8.1.noarch.rp 3.7 MB/s | 251 kB 00:00 2024-04-06T13:55:15.437 INFO:teuthology.orchestra.run.smithi090.stdout:(66/146): python3-websocket-client-0.56.0-5.el8 1.0 MB/s | 61 kB 00:00 2024-04-06T13:55:15.446 INFO:teuthology.orchestra.run.smithi134.stdout:(133/146): libmount-2.32.1-43.el8.x86_64.rpm 1.6 MB/s | 237 kB 00:00 2024-04-06T13:55:15.470 INFO:teuthology.orchestra.run.smithi090.stdout:(67/146): python3-webtest-2.0.33-1.el8.noarch.r 1.4 MB/s | 85 kB 00:00 2024-04-06T13:55:15.496 INFO:teuthology.orchestra.run.smithi090.stdout:(68/146): python3-zc-lockfile-2.0-2.el8.noarch. 391 kB/s | 23 kB 00:00 2024-04-06T13:55:15.503 INFO:teuthology.orchestra.run.smithi105.stdout:(133/146): libmount-2.32.1-43.el8.x86_64.rpm 2.3 MB/s | 237 kB 00:00 2024-04-06T13:55:15.529 INFO:teuthology.orchestra.run.smithi090.stdout:(69/146): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.0 MB/s | 70 kB 00:00 2024-04-06T13:55:15.563 INFO:teuthology.orchestra.run.smithi134.stdout:(134/146): libsmartcols-2.32.1-43.el8.x86_64.rp 1.5 MB/s | 179 kB 00:00 2024-04-06T13:55:15.638 INFO:teuthology.orchestra.run.smithi134.stdout:(135/146): librados2-18.2.2-1086.g052adcad.el8. 5.0 MB/s | 3.7 MB 00:00 2024-04-06T13:55:15.680 INFO:teuthology.orchestra.run.smithi090.stdout:(70/146): thrift-0.13.0-2.el8.x86_64.rpm 8.3 MB/s | 1.7 MB 00:00 2024-04-06T13:55:15.687 INFO:teuthology.orchestra.run.smithi105.stdout:(134/146): libsmartcols-2.32.1-43.el8.x86_64.rp 980 kB/s | 179 kB 00:00 2024-04-06T13:55:15.687 INFO:teuthology.orchestra.run.smithi134.stdout:(136/146): libuuid-2.32.1-43.el8.x86_64.rpm 846 kB/s | 99 kB 00:00 2024-04-06T13:55:15.705 INFO:teuthology.orchestra.run.smithi090.stdout:(71/146): python3-packaging-16.8-9.el8.noarch.r 275 kB/s | 48 kB 00:00 2024-04-06T13:55:15.756 INFO:teuthology.orchestra.run.smithi134.stdout:(137/146): librbd1-18.2.2-1086.g052adcad.el8.x8 4.3 MB/s | 3.6 MB 00:00 2024-04-06T13:55:15.799 INFO:teuthology.orchestra.run.smithi134.stdout:(138/146): util-linux-user-2.32.1-43.el8.x86_64 878 kB/s | 103 kB 00:00 2024-04-06T13:55:15.821 INFO:teuthology.orchestra.run.smithi105.stdout:(135/146): librados2-18.2.2-1086.g052adcad.el8. 5.3 MB/s | 3.7 MB 00:00 2024-04-06T13:55:15.854 INFO:teuthology.orchestra.run.smithi105.stdout:(136/146): libuuid-2.32.1-43.el8.x86_64.rpm 593 kB/s | 99 kB 00:00 2024-04-06T13:55:15.856 INFO:teuthology.orchestra.run.smithi134.stdout:(139/146): util-linux-2.32.1-43.el8.x86_64.rpm 11 MB/s | 2.5 MB 00:00 2024-04-06T13:55:15.890 INFO:teuthology.orchestra.run.smithi134.stdout:(140/146): iproute-6.2.0-5.el8_9.x86_64.rpm 6.2 MB/s | 854 kB 00:00 2024-04-06T13:55:15.914 INFO:teuthology.orchestra.run.smithi090.stdout:(72/146): python3-jsonpointer-1.10-11.el8.noarc 86 kB/s | 20 kB 00:00 2024-04-06T13:55:15.932 INFO:teuthology.orchestra.run.smithi134.stdout:(141/146): libblkid-2.32.1-43.el8.x86_64.rpm 1.6 MB/s | 220 kB 00:00 2024-04-06T13:55:15.939 INFO:teuthology.orchestra.run.smithi090.stdout:(73/146): python3-prettytable-0.7.2-14.el8.noar 193 kB/s | 44 kB 00:00 2024-04-06T13:55:15.963 INFO:teuthology.orchestra.run.smithi105.stdout:(137/146): util-linux-2.32.1-43.el8.x86_64.rpm 17 MB/s | 2.5 MB 00:00 2024-04-06T13:55:15.988 INFO:teuthology.orchestra.run.smithi105.stdout:(138/146): util-linux-user-2.32.1-43.el8.x86_64 766 kB/s | 103 kB 00:00 2024-04-06T13:55:16.016 INFO:teuthology.orchestra.run.smithi134.stdout:(142/146): libfdisk-2.32.1-43.el8.x86_64.rpm 1.6 MB/s | 254 kB 00:00 2024-04-06T13:55:16.031 INFO:teuthology.orchestra.run.smithi090.stdout:(74/146): python3-jsonpatch-1.21-2.el8.noarch.r 290 kB/s | 27 kB 00:00 2024-04-06T13:55:16.047 INFO:teuthology.orchestra.run.smithi105.stdout:(139/146): librbd1-18.2.2-1086.g052adcad.el8.x8 4.0 MB/s | 3.6 MB 00:00 2024-04-06T13:55:16.058 INFO:teuthology.orchestra.run.smithi134.stdout:(143/146): openssl-1.1.1k-12.el8_9.x86_64.rpm 4.2 MB/s | 711 kB 00:00 2024-04-06T13:55:16.089 INFO:teuthology.orchestra.run.smithi105.stdout:(140/146): iproute-6.2.0-5.el8_9.x86_64.rpm 6.6 MB/s | 854 kB 00:00 2024-04-06T13:55:16.090 INFO:teuthology.orchestra.run.smithi090.stdout:(75/146): python3-pyasn1-modules-0.3.7-6.el8.no 627 kB/s | 110 kB 00:00 2024-04-06T13:55:16.116 INFO:teuthology.orchestra.run.smithi134.stdout:(144/146): xfsdump-3.1.8-7.el8_9.x86_64.rpm 3.3 MB/s | 333 kB 00:00 2024-04-06T13:55:16.132 INFO:teuthology.orchestra.run.smithi090.stdout:(76/146): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2024-04-06T13:55:16.139 INFO:teuthology.orchestra.run.smithi105.stdout:(141/146): libblkid-2.32.1-43.el8.x86_64.rpm 1.4 MB/s | 220 kB 00:00 2024-04-06T13:55:16.158 INFO:teuthology.orchestra.run.smithi134.stdout:(145/146): openssl-libs-1.1.1k-12.el8_9.x86_64. 6.5 MB/s | 1.5 MB 00:00 2024-04-06T13:55:16.182 INFO:teuthology.orchestra.run.smithi090.stdout:(77/146): python3-werkzeug-0.12.2-4.el8.noarch. 4.8 MB/s | 457 kB 00:00 2024-04-06T13:55:16.191 INFO:teuthology.orchestra.run.smithi134.stdout:(146/146): xfsprogs-5.0.0-12.el8.x86_64.rpm 8.0 MB/s | 1.1 MB 00:00 2024-04-06T13:55:16.194 INFO:teuthology.orchestra.run.smithi134.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:55:16.195 INFO:teuthology.orchestra.run.smithi134.stdout:Total 10 MB/s | 215 MB 00:21 2024-04-06T13:55:16.203 INFO:teuthology.orchestra.run.smithi105.stdout:(142/146): openssl-1.1.1k-12.el8_9.x86_64.rpm 6.1 MB/s | 711 kB 00:00 2024-04-06T13:55:16.239 INFO:teuthology.orchestra.run.smithi105.stdout:(143/146): libfdisk-2.32.1-43.el8.x86_64.rpm 1.3 MB/s | 254 kB 00:00 2024-04-06T13:55:16.290 INFO:teuthology.orchestra.run.smithi105.stdout:(144/146): openssl-libs-1.1.1k-12.el8_9.x86_64. 9.8 MB/s | 1.5 MB 00:00 2024-04-06T13:55:16.291 INFO:teuthology.orchestra.run.smithi090.stdout:(78/146): bison-3.0.4-10.el8.x86_64.rpm 4.2 MB/s | 688 kB 00:00 2024-04-06T13:55:16.332 INFO:teuthology.orchestra.run.smithi105.stdout:(145/146): xfsdump-3.1.8-7.el8_9.x86_64.rpm 2.5 MB/s | 333 kB 00:00 2024-04-06T13:55:16.358 INFO:teuthology.orchestra.run.smithi090.stdout:(79/146): flex-2.6.1-9.el8.x86_64.rpm 1.8 MB/s | 320 kB 00:00 2024-04-06T13:55:16.423 INFO:teuthology.orchestra.run.smithi105.stdout:(146/146): xfsprogs-5.0.0-12.el8.x86_64.rpm 5.8 MB/s | 1.1 MB 00:00 2024-04-06T13:55:16.427 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:55:16.427 INFO:teuthology.orchestra.run.smithi105.stdout:Total 11 MB/s | 215 MB 00:19 2024-04-06T13:55:16.483 INFO:teuthology.orchestra.run.smithi090.stdout:(80/146): libtool-2.4.6-25.el8.x86_64.rpm 3.6 MB/s | 709 kB 00:00 2024-04-06T13:55:16.508 INFO:teuthology.orchestra.run.smithi090.stdout:(81/146): python3-markupsafe-0.23-19.el8.x86_64 258 kB/s | 39 kB 00:00 2024-04-06T13:55:16.609 INFO:teuthology.orchestra.run.smithi090.stdout:(82/146): uuid-1.6.2-43.el8.x86_64.rpm 639 kB/s | 64 kB 00:00 2024-04-06T13:55:16.643 INFO:teuthology.orchestra.run.smithi090.stdout:(83/146): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.6 MB/s | 259 kB 00:00 2024-04-06T13:55:16.768 INFO:teuthology.orchestra.run.smithi090.stdout:(84/146): python3-pyOpenSSL-19.0.0-1.el8.noarch 653 kB/s | 103 kB 00:00 2024-04-06T13:55:16.893 INFO:teuthology.orchestra.run.smithi090.stdout:(85/146): python3-babel-2.5.1-7.el8.noarch.rpm 19 MB/s | 4.8 MB 00:00 2024-04-06T13:55:16.928 INFO:teuthology.orchestra.run.smithi090.stdout:(86/146): python3-jinja2-2.10.1-3.el8.noarch.rp 3.3 MB/s | 538 kB 00:00 2024-04-06T13:55:17.235 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction check 2024-04-06T13:55:17.477 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-04-06T13:55:17.564 INFO:teuthology.orchestra.run.smithi090.stdout:(87/146): ceph-test-18.2.2-1086.g052adcad.el8.x 4.7 MB/s | 47 MB 00:10 2024-04-06T13:55:17.596 INFO:teuthology.orchestra.run.smithi090.stdout:(88/146): socat-1.7.4.1-1.el8.x86_64.rpm 484 kB/s | 323 kB 00:00 2024-04-06T13:55:17.628 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction check succeeded. 2024-04-06T13:55:17.628 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction test 2024-04-06T13:55:17.688 INFO:teuthology.orchestra.run.smithi090.stdout:(89/146): python3-numpy-1.14.3-10.el8.x86_64.rp 4.7 MB/s | 3.7 MB 00:00 2024-04-06T13:55:17.788 INFO:teuthology.orchestra.run.smithi090.stdout:(90/146): python3-numpy-f2py-1.14.3-10.el8.x86_ 1.0 MB/s | 225 kB 00:00 2024-04-06T13:55:17.830 INFO:teuthology.orchestra.run.smithi090.stdout:(91/146): librdkafka-0.11.4-3.el8.x86_64.rpm 1.5 MB/s | 354 kB 00:00 2024-04-06T13:55:17.871 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-04-06T13:55:17.871 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-04-06T13:55:17.972 INFO:teuthology.orchestra.run.smithi090.stdout:(92/146): python3-mako-1.0.6-14.el8.noarch.rpm 855 kB/s | 157 kB 00:00 2024-04-06T13:55:18.072 INFO:teuthology.orchestra.run.smithi090.stdout:(93/146): openblas-threads-0.3.15-6.el8.x86_64. 14 MB/s | 5.2 MB 00:00 2024-04-06T13:55:18.139 INFO:teuthology.orchestra.run.smithi090.stdout:(94/146): automake-1.16.1-8.el8.noarch.rpm 4.2 MB/s | 714 kB 00:00 2024-04-06T13:55:18.173 INFO:teuthology.orchestra.run.smithi090.stdout:(95/146): libpq-13.11-1.el8.x86_64.rpm 1.9 MB/s | 198 kB 00:00 2024-04-06T13:55:18.248 INFO:teuthology.orchestra.run.smithi090.stdout:(96/146): jq-1.6-7.el8.x86_64.rpm 1.8 MB/s | 202 kB 00:00 2024-04-06T13:55:18.273 INFO:teuthology.orchestra.run.smithi090.stdout:(97/146): python3-pytz-2017.2-11.el8.noarch.rpm 541 kB/s | 54 kB 00:00 2024-04-06T13:55:18.357 INFO:teuthology.orchestra.run.smithi090.stdout:(98/146): openblas-0.3.15-6.el8.x86_64.rpm 9.6 MB/s | 5.0 MB 00:00 2024-04-06T13:55:18.408 INFO:teuthology.orchestra.run.smithi090.stdout:(99/146): oniguruma-6.8.2-2.1.el8_9.x86_64.rpm 1.4 MB/s | 188 kB 00:00 2024-04-06T13:55:18.525 INFO:teuthology.orchestra.run.smithi090.stdout:(100/146): postgresql-contrib-10.23-4.module+el 6.8 MB/s | 812 kB 00:00 2024-04-06T13:55:18.575 INFO:teuthology.orchestra.run.smithi090.stdout:(101/146): postgresql-10.23-4.module+el8.9.0+21 6.9 MB/s | 1.5 MB 00:00 2024-04-06T13:55:18.784 INFO:teuthology.orchestra.run.smithi090.stdout:(102/146): python3-scipy-1.0.0-21.module+el8.9. 26 MB/s | 14 MB 00:00 2024-04-06T13:55:18.809 INFO:teuthology.orchestra.run.smithi090.stdout:(103/146): libverto-devel-0.3.0-5.el8.x86_64.rp 76 kB/s | 18 kB 00:00 2024-04-06T13:55:18.830 INFO:teuthology.orchestra.run.smithi134.stdout:Transaction test succeeded. 2024-04-06T13:55:18.840 INFO:teuthology.orchestra.run.smithi134.stdout:Running transaction 2024-04-06T13:55:18.893 INFO:teuthology.orchestra.run.smithi090.stdout:(104/146): python3-jwt-1.6.1-2.el8.noarch.rpm 393 kB/s | 43 kB 00:00 2024-04-06T13:55:18.976 INFO:teuthology.orchestra.run.smithi090.stdout:(105/146): postgresql-server-10.23-4.module+el8 11 MB/s | 5.1 MB 00:00 2024-04-06T13:55:19.002 INFO:teuthology.orchestra.run.smithi090.stdout:(106/146): python3-pycparser-2.14-14.el8.noarch 1.0 MB/s | 109 kB 00:00 2024-04-06T13:55:19.056 INFO:teuthology.orchestra.run.smithi090.stdout:(107/146): python3-oauthlib-2.1.0-1.el8.noarch. 626 kB/s | 155 kB 00:00 2024-04-06T13:55:19.070 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-04-06T13:55:19.082 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-04-06T13:55:19.085 INFO:teuthology.orchestra.run.smithi090.stdout:(108/146): python3-requests-oauthlib-1.0.0-1.el 391 kB/s | 43 kB 00:00 2024-04-06T13:55:19.119 INFO:teuthology.orchestra.run.smithi090.stdout:(109/146): libaio-devel-0.3.112-1.el8.x86_64.rp 159 kB/s | 19 kB 00:00 2024-04-06T13:55:19.194 INFO:teuthology.orchestra.run.smithi090.stdout:(110/146): libzstd-devel-1.4.4-1.el8.x86_64.rpm 318 kB/s | 44 kB 00:00 2024-04-06T13:55:19.228 INFO:teuthology.orchestra.run.smithi090.stdout:(111/146): pcre2-devel-10.32-2.el8.x86_64.rpm 4.2 MB/s | 605 kB 00:00 2024-04-06T13:55:19.263 INFO:teuthology.orchestra.run.smithi090.stdout:(112/146): pcre2-utf16-10.32-2.el8.x86_64.rpm 1.6 MB/s | 229 kB 00:00 2024-04-06T13:55:19.372 INFO:teuthology.orchestra.run.smithi090.stdout:(113/146): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.2 MB/s | 220 kB 00:00 2024-04-06T13:55:19.430 INFO:teuthology.orchestra.run.smithi090.stdout:(114/146): libselinux-devel-2.9-5.el8.x86_64.rp 990 kB/s | 200 kB 00:00 2024-04-06T13:55:19.489 INFO:teuthology.orchestra.run.smithi090.stdout:(115/146): libsepol-devel-2.9-3.el8.x86_64.rpm 742 kB/s | 87 kB 00:00 2024-04-06T13:55:19.514 INFO:teuthology.orchestra.run.smithi090.stdout:(116/146): keyutils-libs-devel-1.5.10-9.el8.x86 192 kB/s | 48 kB 00:00 2024-04-06T13:55:19.548 INFO:teuthology.orchestra.run.smithi090.stdout:(117/146): zlib-devel-1.2.11-18.el8_5.x86_64.rp 491 kB/s | 58 kB 00:00 2024-04-06T13:55:19.640 INFO:teuthology.orchestra.run.smithi090.stdout:(118/146): fuse-2.9.7-17.el8.x86_64.rpm 662 kB/s | 83 kB 00:00 2024-04-06T13:55:19.657 INFO:teuthology.orchestra.run.smithi090.stdout:(119/146): libcom_err-devel-1.45.6-4.el8.x86_64 230 kB/s | 39 kB 00:00 2024-04-06T13:55:19.718 INFO:teuthology.orchestra.run.smithi090.stdout:(120/146): libgfortran-8.5.0-20.el8.x86_64.rpm 3.8 MB/s | 645 kB 00:00 2024-04-06T13:55:19.774 INFO:teuthology.orchestra.run.smithi090.stdout:(121/146): libquadmath-8.5.0-20.el8.x86_64.rpm 1.3 MB/s | 172 kB 00:00 2024-04-06T13:55:19.800 INFO:teuthology.orchestra.run.smithi090.stdout:(122/146): libuuid-devel-2.32.1-43.el8.x86_64.r 701 kB/s | 100 kB 00:00 2024-04-06T13:55:19.825 INFO:teuthology.orchestra.run.smithi090.stdout:(123/146): python3-cffi-1.11.5-6.el8.x86_64.rpm 2.1 MB/s | 238 kB 00:00 2024-04-06T13:55:19.884 INFO:teuthology.orchestra.run.smithi090.stdout:(124/146): xfsprogs-devel-5.0.0-12.el8.x86_64.r 667 kB/s | 73 kB 00:00 2024-04-06T13:55:19.942 INFO:teuthology.orchestra.run.smithi090.stdout:(125/146): NetworkManager-initscripts-updown-1. 1.0 MB/s | 144 kB 00:00 2024-04-06T13:55:19.968 INFO:teuthology.orchestra.run.smithi090.stdout:(126/146): elfutils-libelf-devel-0.189-3.el8.x8 433 kB/s | 62 kB 00:00 2024-04-06T13:55:20.026 INFO:teuthology.orchestra.run.smithi090.stdout:(127/146): krb5-devel-1.18.2-26.el8_9.x86_64.rp 3.9 MB/s | 562 kB 00:00 2024-04-06T13:55:20.110 INFO:teuthology.orchestra.run.smithi090.stdout:(128/146): openssl-devel-1.1.1k-12.el8_9.x86_64 14 MB/s | 2.3 MB 00:00 2024-04-06T13:55:20.143 INFO:teuthology.orchestra.run.smithi090.stdout:(129/146): librabbitmq-0.9.0-5.el8_9.x86_64.rpm 268 kB/s | 47 kB 00:00 2024-04-06T13:55:20.202 INFO:teuthology.orchestra.run.smithi090.stdout:(130/146): python3-cryptography-3.2.1-7.el8_9.x 3.1 MB/s | 559 kB 00:00 2024-04-06T13:55:20.252 INFO:teuthology.orchestra.run.smithi090.stdout:(131/146): elfutils-libelf-0.189-3.el8.x86_64.r 4.5 MB/s | 232 kB 00:00 2024-04-06T13:55:20.370 INFO:teuthology.orchestra.run.smithi090.stdout:(132/146): elfutils-libs-0.189-3.el8.x86_64.rpm 2.5 MB/s | 303 kB 00:00 2024-04-06T13:55:20.520 INFO:teuthology.orchestra.run.smithi090.stdout:(133/146): libmount-2.32.1-43.el8.x86_64.rpm 1.5 MB/s | 237 kB 00:00 2024-04-06T13:55:20.670 INFO:teuthology.orchestra.run.smithi090.stdout:(134/146): libsmartcols-2.32.1-43.el8.x86_64.rp 1.2 MB/s | 179 kB 00:00 2024-04-06T13:55:20.820 INFO:teuthology.orchestra.run.smithi090.stdout:(135/146): libuuid-2.32.1-43.el8.x86_64.rpm 661 kB/s | 99 kB 00:00 2024-04-06T13:55:20.921 INFO:teuthology.orchestra.run.smithi090.stdout:(136/146): librados2-18.2.2-1086.g052adcad.el8. 4.6 MB/s | 3.7 MB 00:00 2024-04-06T13:55:20.997 INFO:teuthology.orchestra.run.smithi090.stdout:(137/146): util-linux-2.32.1-43.el8.x86_64.rpm 14 MB/s | 2.5 MB 00:00 2024-04-06T13:55:21.056 INFO:teuthology.orchestra.run.smithi090.stdout:(138/146): util-linux-user-2.32.1-43.el8.x86_64 762 kB/s | 103 kB 00:00 2024-04-06T13:55:21.115 INFO:teuthology.orchestra.run.smithi090.stdout:(139/146): iproute-6.2.0-5.el8_9.x86_64.rpm 7.1 MB/s | 854 kB 00:00 2024-04-06T13:55:21.240 INFO:teuthology.orchestra.run.smithi090.stdout:(140/146): librbd1-18.2.2-1086.g052adcad.el8.x8 3.3 MB/s | 3.6 MB 00:01 2024-04-06T13:55:21.274 INFO:teuthology.orchestra.run.smithi090.stdout:(141/146): libblkid-2.32.1-43.el8.x86_64.rpm 1.0 MB/s | 220 kB 00:00 2024-04-06T13:55:21.299 INFO:teuthology.orchestra.run.smithi090.stdout:(142/146): libfdisk-2.32.1-43.el8.x86_64.rpm 1.3 MB/s | 254 kB 00:00 2024-04-06T13:55:21.366 INFO:teuthology.orchestra.run.smithi090.stdout:(143/146): openssl-1.1.1k-12.el8_9.x86_64.rpm 5.5 MB/s | 711 kB 00:00 2024-04-06T13:55:21.433 INFO:teuthology.orchestra.run.smithi090.stdout:(144/146): xfsdump-3.1.8-7.el8_9.x86_64.rpm 2.4 MB/s | 333 kB 00:00 2024-04-06T13:55:21.491 INFO:teuthology.orchestra.run.smithi090.stdout:(145/146): openssl-libs-1.1.1k-12.el8_9.x86_64. 6.8 MB/s | 1.5 MB 00:00 2024-04-06T13:55:21.542 INFO:teuthology.orchestra.run.smithi090.stdout:(146/146): xfsprogs-5.0.0-12.el8.x86_64.rpm 6.1 MB/s | 1.1 MB 00:00 2024-04-06T13:55:21.546 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:55:21.546 INFO:teuthology.orchestra.run.smithi090.stdout:Total 9.9 MB/s | 215 MB 00:21 2024-04-06T13:55:22.581 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction check 2024-04-06T13:55:22.974 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction check succeeded. 2024-04-06T13:55:22.974 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction test 2024-04-06T13:55:24.192 INFO:teuthology.orchestra.run.smithi090.stdout:Transaction test succeeded. 2024-04-06T13:55:24.202 INFO:teuthology.orchestra.run.smithi090.stdout:Running transaction 2024-04-06T13:55:24.868 INFO:teuthology.orchestra.run.smithi134.stdout: Preparing : 1/1 2024-04-06T13:55:25.002 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/1 2024-04-06T13:55:25.211 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-04-06T13:55:25.272 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:25.345 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/1 2024-04-06T13:55:25.489 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:25.595 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:25.660 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : thrift-0.13.0-2.el8.x86_64 2/162 2024-04-06T13:55:25.847 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:25.875 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:25.951 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:26.108 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : thrift-0.13.0-2.el8.x86_64 2/162 2024-04-06T13:55:26.125 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:26.229 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:26.257 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:26.359 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:26.456 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : liboath-2.6.2-3.el8.x86_64 5/162 2024-04-06T13:55:26.508 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:26.586 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:26.667 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:26.729 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:26.822 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : liboath-2.6.2-3.el8.x86_64 5/162 2024-04-06T13:55:26.943 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librabbitmq-0.9.0-5.el8_9.x86_64 7/162 2024-04-06T13:55:27.008 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:27.062 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:27.116 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:27.224 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librabbitmq-0.9.0-5.el8_9.x86_64 7/162 2024-04-06T13:55:27.399 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:27.507 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:27.741 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:27.798 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:28.016 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:28.132 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:28.354 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:28.400 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:28.480 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:28.602 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:28.691 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:28.729 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:28.774 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:28.916 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:29.009 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:29.067 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:55:29.309 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:55:29.416 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:29.499 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:29.641 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:29.697 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libpq-13.11-1.el8.x86_64 14/162 2024-04-06T13:55:29.725 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:29.953 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libpq-13.11-1.el8.x86_64 14/162 2024-04-06T13:55:29.961 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : elfutils-libelf-0.189-3.el8.x86_64 15/162 2024-04-06T13:55:30.026 INFO:teuthology.orchestra.run.smithi090.stdout: Preparing : 1/1 2024-04-06T13:55:30.160 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/1 2024-04-06T13:55:30.194 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 16/162 2024-04-06T13:55:30.202 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : elfutils-libelf-0.189-3.el8.x86_64 15/162 2024-04-06T13:55:30.379 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:30.410 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 16/162 2024-04-06T13:55:30.577 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 17/162 2024-04-06T13:55:30.591 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: openssl-libs-1:1.1.1k-12.el8_9.x86_64 1/162 2024-04-06T13:55:30.777 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : thrift-0.13.0-2.el8.x86_64 2/162 2024-04-06T13:55:30.827 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 17/162 2024-04-06T13:55:30.853 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 18/162 2024-04-06T13:55:30.952 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:31.008 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:31.030 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libuuid-2.32.1-43.el8.x86_64 3/162 2024-04-06T13:55:31.053 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 18/162 2024-04-06T13:55:31.143 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:31.224 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:31.227 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:31.291 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:31.299 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libblkid-2.32.1-43.el8.x86_64 4/162 2024-04-06T13:55:31.327 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:31.356 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:31.490 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:31.508 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : liboath-2.6.2-3.el8.x86_64 5/162 2024-04-06T13:55:31.532 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:31.566 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:31.658 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:31.694 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:31.715 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:31.758 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 6/162 2024-04-06T13:55:31.851 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:31.880 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:31.988 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librabbitmq-0.9.0-5.el8_9.x86_64 7/162 2024-04-06T13:55:32.036 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:32.168 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:32.472 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:32.572 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 8/162 2024-04-06T13:55:32.682 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:32.683 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:32.794 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:32.840 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:33.189 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:55:33.290 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:33.409 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:33.448 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:33.482 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:33.535 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:55:33.653 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:33.727 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:33.847 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:55:34.161 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:55:34.181 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:34.326 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:34.460 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:34.471 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 25/162 2024-04-06T13:55:34.547 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: xfsprogs-5.0.0-12.el8.x86_64 13/162 2024-04-06T13:55:34.596 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 25/162 2024-04-06T13:55:34.624 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 26/162 2024-04-06T13:55:34.728 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 26/162 2024-04-06T13:55:34.767 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libpq-13.11-1.el8.x86_64 14/162 2024-04-06T13:55:34.785 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 27/162 2024-04-06T13:55:34.910 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 27/162 2024-04-06T13:55:35.023 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : elfutils-libelf-0.189-3.el8.x86_64 15/162 2024-04-06T13:55:35.107 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 28/162 2024-04-06T13:55:35.214 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 28/162 2024-04-06T13:55:35.222 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 16/162 2024-04-06T13:55:35.437 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 29/162 2024-04-06T13:55:35.512 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 29/162 2024-04-06T13:55:35.605 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 17/162 2024-04-06T13:55:35.683 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 30/162 2024-04-06T13:55:35.791 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 30/162 2024-04-06T13:55:35.864 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 18/162 2024-04-06T13:55:35.877 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 31/162 2024-04-06T13:55:35.969 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 31/162 2024-04-06T13:55:36.010 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:36.051 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 32/162 2024-04-06T13:55:36.101 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:55:36.134 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 32/162 2024-04-06T13:55:36.252 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:36.300 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 33/162 2024-04-06T13:55:36.343 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 33/162 2024-04-06T13:55:36.357 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 20/162 2024-04-06T13:55:36.518 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:36.649 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 21/162 2024-04-06T13:55:36.795 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:37.266 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 34/162 2024-04-06T13:55:37.283 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 34/162 2024-04-06T13:55:37.419 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 22/162 2024-04-06T13:55:37.594 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 35/162 2024-04-06T13:55:37.627 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:37.645 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 35/162 2024-04-06T13:55:37.893 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 36/162 2024-04-06T13:55:37.944 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 36/162 2024-04-06T13:55:38.056 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:38.156 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:38.225 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: openblas-threads-0.3.15-6.el8.x86_64 23/162 2024-04-06T13:55:38.427 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:39.113 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: openblas-0.3.15-6.el8.x86_64 24/162 2024-04-06T13:55:39.383 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 25/162 2024-04-06T13:55:39.574 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 26/162 2024-04-06T13:55:39.747 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 27/162 2024-04-06T13:55:39.797 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:39.891 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:40.012 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 28/162 2024-04-06T13:55:40.140 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 38/162 2024-04-06T13:55:40.198 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 38/162 2024-04-06T13:55:40.290 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 29/162 2024-04-06T13:55:40.384 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rsa-4.9-2.el8.noarch 39/162 2024-04-06T13:55:40.443 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rsa-4.9-2.el8.noarch 39/162 2024-04-06T13:55:40.519 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 30/162 2024-04-06T13:55:40.621 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 40/162 2024-04-06T13:55:40.698 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 31/162 2024-04-06T13:55:40.700 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 40/162 2024-04-06T13:55:40.791 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 41/162 2024-04-06T13:55:40.854 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 32/162 2024-04-06T13:55:40.895 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 41/162 2024-04-06T13:55:40.983 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 42/162 2024-04-06T13:55:41.087 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 42/162 2024-04-06T13:55:41.114 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 33/162 2024-04-06T13:55:41.216 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:41.292 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:41.332 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:41.409 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:41.530 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:41.601 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:41.673 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:41.743 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:41.867 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libuuid-devel-2.32.1-43.el8.x86_64 45/162 2024-04-06T13:55:41.952 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libuuid-devel-2.32.1-43.el8.x86_64 45/162 2024-04-06T13:55:42.034 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 34/162 2024-04-06T13:55:42.066 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 46/162 2024-04-06T13:55:42.175 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 46/162 2024-04-06T13:55:42.281 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:42.422 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:42.423 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 35/162 2024-04-06T13:55:42.663 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 36/162 2024-04-06T13:55:42.695 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:42.851 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:42.966 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:43.047 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:43.072 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:43.248 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-packaging-16.8-9.el8.noarch 49/162 2024-04-06T13:55:43.423 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:43.440 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:43.451 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : fuse-2.9.7-17.el8.x86_64 50/162 2024-04-06T13:55:43.668 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : fuse-2.9.7-17.el8.x86_64 49/162 2024-04-06T13:55:43.692 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 51/162 2024-04-06T13:55:43.850 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 52/162 2024-04-06T13:55:43.892 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 50/162 2024-04-06T13:55:44.071 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 53/162 2024-04-06T13:55:44.076 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 51/162 2024-04-06T13:55:44.242 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 54/162 2024-04-06T13:55:44.267 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 52/162 2024-04-06T13:55:44.459 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 53/162 2024-04-06T13:55:44.536 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 55/162 2024-04-06T13:55:44.588 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 37/162 2024-04-06T13:55:44.777 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 54/162 2024-04-06T13:55:44.809 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 56/162 2024-04-06T13:55:44.969 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 38/162 2024-04-06T13:55:45.052 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 57/162 2024-04-06T13:55:45.086 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 55/162 2024-04-06T13:55:45.188 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rsa-4.9-2.el8.noarch 39/162 2024-04-06T13:55:45.192 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libzstd-devel-1.4.4-1.el8.x86_64 58/162 2024-04-06T13:55:45.336 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 56/162 2024-04-06T13:55:45.424 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 59/162 2024-04-06T13:55:45.445 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 40/162 2024-04-06T13:55:45.509 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libzstd-devel-1.4.4-1.el8.x86_64 57/162 2024-04-06T13:55:45.632 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 41/162 2024-04-06T13:55:45.783 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 58/162 2024-04-06T13:55:45.791 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 42/162 2024-04-06T13:55:45.803 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 60/162 2024-04-06T13:55:46.026 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:46.100 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cryptography-3.2.1-7.el8_9.x86_64 61/162 2024-04-06T13:55:46.103 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libmount-2.32.1-43.el8.x86_64 43/162 2024-04-06T13:55:46.171 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 59/162 2024-04-06T13:55:46.318 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:46.387 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libfdisk-2.32.1-43.el8.x86_64 44/162 2024-04-06T13:55:46.413 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 62/162 2024-04-06T13:55:46.508 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cryptography-3.2.1-7.el8_9.x86_64 60/162 2024-04-06T13:55:46.629 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 63/162 2024-04-06T13:55:46.664 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libuuid-devel-2.32.1-43.el8.x86_64 45/162 2024-04-06T13:55:46.830 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 64/162 2024-04-06T13:55:46.838 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 61/162 2024-04-06T13:55:46.853 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 46/162 2024-04-06T13:55:47.051 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:47.088 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 62/162 2024-04-06T13:55:47.120 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 65/162 2024-04-06T13:55:47.282 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 63/162 2024-04-06T13:55:47.393 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 66/162 2024-04-06T13:55:47.558 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 64/162 2024-04-06T13:55:47.569 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libsmartcols-2.32.1-43.el8.x86_64 47/162 2024-04-06T13:55:47.597 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 67/162 2024-04-06T13:55:47.803 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 68/162 2024-04-06T13:55:47.827 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 65/162 2024-04-06T13:55:47.877 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:47.894 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: util-linux-2.32.1-43.el8.x86_64 48/162 2024-04-06T13:55:48.018 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 69/162 2024-04-06T13:55:48.040 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 66/162 2024-04-06T13:55:48.113 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : fuse-2.9.7-17.el8.x86_64 49/162 2024-04-06T13:55:48.246 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : krb5-devel-1.18.2-26.el8_9.x86_64 70/162 2024-04-06T13:55:48.266 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 67/162 2024-04-06T13:55:48.298 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 50/162 2024-04-06T13:55:48.413 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : oniguruma-6.8.2-2.1.el8_9.x86_64 71/162 2024-04-06T13:55:48.456 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 51/162 2024-04-06T13:55:48.492 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: oniguruma-6.8.2-2.1.el8_9.x86_64 71/162 2024-04-06T13:55:48.563 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 68/162 2024-04-06T13:55:48.655 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 52/162 2024-04-06T13:55:48.790 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : jq-1.6-7.el8.x86_64 72/162 2024-04-06T13:55:48.799 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : krb5-devel-1.18.2-26.el8_9.x86_64 69/162 2024-04-06T13:55:48.838 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 53/162 2024-04-06T13:55:48.992 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-packaging-16.8-9.el8.noarch 70/162 2024-04-06T13:55:49.060 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : automake-1.16.1-8.el8.noarch 73/162 2024-04-06T13:55:49.131 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 54/162 2024-04-06T13:55:49.197 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : oniguruma-6.8.2-2.1.el8_9.x86_64 71/162 2024-04-06T13:55:49.238 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : uuid-1.6.2-43.el8.x86_64 74/162 2024-04-06T13:55:49.278 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: oniguruma-6.8.2-2.1.el8_9.x86_64 71/162 2024-04-06T13:55:49.303 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 74/162 2024-04-06T13:55:49.406 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 55/162 2024-04-06T13:55:49.499 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 75/162 2024-04-06T13:55:49.555 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : jq-1.6-7.el8.x86_64 72/162 2024-04-06T13:55:49.614 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 56/162 2024-04-06T13:55:49.653 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 76/162 2024-04-06T13:55:49.812 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : automake-1.16.1-8.el8.noarch 73/162 2024-04-06T13:55:49.814 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libzstd-devel-1.4.4-1.el8.x86_64 57/162 2024-04-06T13:55:49.873 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 77/162 2024-04-06T13:55:50.005 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 78/162 2024-04-06T13:55:50.029 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 58/162 2024-04-06T13:55:50.038 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : uuid-1.6.2-43.el8.x86_64 74/162 2024-04-06T13:55:50.098 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 74/162 2024-04-06T13:55:50.214 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 79/162 2024-04-06T13:55:50.312 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 75/162 2024-04-06T13:55:50.374 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 59/162 2024-04-06T13:55:50.496 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 76/162 2024-04-06T13:55:50.542 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 80/162 2024-04-06T13:55:50.671 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cryptography-3.2.1-7.el8_9.x86_64 60/162 2024-04-06T13:55:50.731 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 77/162 2024-04-06T13:55:50.786 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 81/162 2024-04-06T13:55:50.888 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 78/162 2024-04-06T13:55:50.950 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 61/162 2024-04-06T13:55:50.950 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 82/162 2024-04-06T13:55:51.093 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 79/162 2024-04-06T13:55:51.142 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 83/162 2024-04-06T13:55:51.176 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 62/162 2024-04-06T13:55:51.371 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 63/162 2024-04-06T13:55:51.387 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 84/162 2024-04-06T13:55:51.411 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 80/162 2024-04-06T13:55:51.609 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 85/162 2024-04-06T13:55:51.617 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 64/162 2024-04-06T13:55:51.662 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 81/162 2024-04-06T13:55:51.834 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 82/162 2024-04-06T13:55:51.865 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 86/162 2024-04-06T13:55:51.880 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 65/162 2024-04-06T13:55:52.025 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 83/162 2024-04-06T13:55:52.062 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 87/162 2024-04-06T13:55:52.068 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 66/162 2024-04-06T13:55:52.193 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 88/162 2024-04-06T13:55:52.231 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 84/162 2024-04-06T13:55:52.278 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 67/162 2024-04-06T13:55:52.432 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 89/162 2024-04-06T13:55:52.490 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 68/162 2024-04-06T13:55:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 85/162 2024-04-06T13:55:52.649 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 90/162 2024-04-06T13:55:52.704 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : krb5-devel-1.18.2-26.el8_9.x86_64 69/162 2024-04-06T13:55:52.774 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 86/162 2024-04-06T13:55:52.826 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 91/162 2024-04-06T13:55:52.893 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : oniguruma-6.8.2-2.1.el8_9.x86_64 70/162 2024-04-06T13:55:52.958 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 87/162 2024-04-06T13:55:52.991 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: oniguruma-6.8.2-2.1.el8_9.x86_64 70/162 2024-04-06T13:55:53.133 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-portend-2.6-1.el8.noarch 92/162 2024-04-06T13:55:53.143 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 88/162 2024-04-06T13:55:53.299 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : jq-1.6-7.el8.x86_64 71/162 2024-04-06T13:55:53.359 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 89/162 2024-04-06T13:55:53.387 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 93/162 2024-04-06T13:55:53.490 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : automake-1.16.1-8.el8.noarch 72/162 2024-04-06T13:55:53.583 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 90/162 2024-04-06T13:55:53.603 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 94/162 2024-04-06T13:55:53.684 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : uuid-1.6.2-43.el8.x86_64 73/162 2024-04-06T13:55:53.751 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 73/162 2024-04-06T13:55:53.785 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 91/162 2024-04-06T13:55:53.870 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 95/162 2024-04-06T13:55:53.992 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 74/162 2024-04-06T13:55:54.041 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-portend-2.6-1.el8.noarch 92/162 2024-04-06T13:55:54.192 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 75/162 2024-04-06T13:55:54.278 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 93/162 2024-04-06T13:55:54.429 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 76/162 2024-04-06T13:55:54.478 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 94/162 2024-04-06T13:55:54.636 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-packaging-16.8-9.el8.noarch 77/162 2024-04-06T13:55:54.695 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 95/162 2024-04-06T13:55:54.810 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 78/162 2024-04-06T13:55:55.001 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 96/162 2024-04-06T13:55:55.031 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 79/162 2024-04-06T13:55:55.334 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 97/162 2024-04-06T13:55:55.334 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 80/162 2024-04-06T13:55:55.550 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 81/162 2024-04-06T13:55:55.561 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 98/162 2024-04-06T13:55:55.731 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 82/162 2024-04-06T13:55:55.800 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 96/162 2024-04-06T13:55:55.870 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 99/162 2024-04-06T13:55:55.888 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 83/162 2024-04-06T13:55:56.089 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 84/162 2024-04-06T13:55:56.099 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 97/162 2024-04-06T13:55:56.134 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 100/162 2024-04-06T13:55:56.314 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 85/162 2024-04-06T13:55:56.361 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 98/162 2024-04-06T13:55:56.378 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 101/162 2024-04-06T13:55:56.546 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 86/162 2024-04-06T13:55:56.671 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 99/162 2024-04-06T13:55:56.738 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 87/162 2024-04-06T13:55:56.932 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 88/162 2024-04-06T13:55:56.938 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 100/162 2024-04-06T13:55:57.137 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 101/162 2024-04-06T13:55:57.155 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 89/162 2024-04-06T13:55:57.346 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 90/162 2024-04-06T13:55:57.557 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 91/162 2024-04-06T13:55:57.705 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 102/162 2024-04-06T13:55:57.813 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-portend-2.6-1.el8.noarch 92/162 2024-04-06T13:55:57.887 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:55:57.952 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 102/162 2024-04-06T13:55:57.957 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:55:58.112 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:55:58.112 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 104/162 2024-04-06T13:55:58.150 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 93/162 2024-04-06T13:55:58.184 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:55:58.370 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 104/162 2024-04-06T13:55:58.433 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 94/162 2024-04-06T13:55:58.675 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 95/162 2024-04-06T13:56:00.132 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 96/162 2024-04-06T13:56:00.580 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 97/162 2024-04-06T13:56:00.834 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 98/162 2024-04-06T13:56:01.253 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 99/162 2024-04-06T13:56:01.493 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 100/162 2024-04-06T13:56:01.802 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:01.817 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 101/162 2024-04-06T13:56:02.045 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:02.105 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:02.361 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:02.621 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 102/162 2024-04-06T13:56:02.638 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:02.951 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:02.959 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:56:03.031 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 103/162 2024-04-06T13:56:03.140 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:03.229 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:03.235 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 104/162 2024-04-06T13:56:03.249 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:03.323 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:03.395 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:03.511 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 105/162 2024-04-06T13:56:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 106/162 2024-04-06T13:56:09.167 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /sys 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /proc 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /dev 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /run 2024-04-06T13:56:22.819 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /mnt 2024-04-06T13:56:22.820 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /var/tmp 2024-04-06T13:56:22.820 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /home 2024-04-06T13:56:22.820 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /tmp 2024-04-06T13:56:22.820 INFO:teuthology.orchestra.run.smithi134.stdout:skipping the directory /dev 2024-04-06T13:56:22.820 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:56:22.830 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /sys 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /proc 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /dev 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /run 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /mnt 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /var/tmp 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /home 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /tmp 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /dev 2024-04-06T13:56:22.831 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:56:23.192 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:23.239 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:24.827 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:24.869 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:25.037 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:25.079 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:25.256 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:25.296 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:25.466 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:25.475 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:25.941 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:25.950 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:26.092 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:26.109 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:26.149 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:26.167 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:27.857 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:27.949 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:27.997 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:28.090 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:28.179 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 113/162 2024-04-06T13:56:28.273 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 113/162 2024-04-06T13:56:28.338 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 114/162 2024-04-06T13:56:28.449 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 114/162 2024-04-06T13:56:28.550 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:28.707 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 107/162 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /sys 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /proc 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /dev 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /run 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /mnt 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /var/tmp 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /home 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /tmp 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout:skipping the directory /dev 2024-04-06T13:56:29.229 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:56:29.579 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:29.583 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:29.750 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:29.811 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:30.036 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:30.777 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:30.941 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:31.016 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:31.033 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:31.173 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:31.215 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 108/162 2024-04-06T13:56:31.267 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:31.410 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:31.415 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 118/162 2024-04-06T13:56:31.578 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:31.616 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:31.627 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 109/162 2024-04-06T13:56:31.668 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 118/162 2024-04-06T13:56:31.833 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:31.837 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:31.875 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:31.969 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:31.981 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:32.040 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:32.052 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:32.306 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 110/162 2024-04-06T13:56:32.541 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:32.559 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 111/162 2024-04-06T13:56:33.283 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 121/162 2024-04-06T13:56:33.321 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 121/162 2024-04-06T13:56:33.469 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:33.504 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:33.553 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:33.587 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:33.706 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:33.749 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:33.801 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:33.844 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:33.929 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:34.007 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:34.276 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:34.333 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:34.354 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:34.441 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:34.521 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 112/162 2024-04-06T13:56:34.556 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:34.654 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 113/162 2024-04-06T13:56:34.825 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 114/162 2024-04-06T13:56:35.074 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:36.100 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 115/162 2024-04-06T13:56:36.342 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:37.306 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 116/162 2024-04-06T13:56:37.466 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:37.558 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 117/162 2024-04-06T13:56:37.881 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 118/162 2024-04-06T13:56:38.043 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:38.081 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 119/162 2024-04-06T13:56:38.266 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:38.277 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 120/162 2024-04-06T13:56:38.886 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:39.044 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:39.389 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 126/162 2024-04-06T13:56:39.522 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 121/162 2024-04-06T13:56:39.547 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 126/162 2024-04-06T13:56:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 127/162 2024-04-06T13:56:39.734 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:39.769 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 122/162 2024-04-06T13:56:39.894 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 127/162 2024-04-06T13:56:39.923 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:39.967 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:40.010 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.2-1086.g052ad 123/162 2024-04-06T13:56:40.122 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:40.216 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:40.563 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 124/162 2024-04-06T13:56:40.595 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:40.791 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:40.802 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:40.921 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : openssl-devel-1:1.1.1k-12.el8_9.x86_64 129/162 2024-04-06T13:56:41.126 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : openssl-devel-1:1.1.1k-12.el8_9.x86_64 129/162 2024-04-06T13:56:41.193 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : elfutils-libelf-devel-0.189-3.el8.x86_64 130/162 2024-04-06T13:56:41.388 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : elfutils-libelf-devel-0.189-3.el8.x86_64 130/162 2024-04-06T13:56:41.420 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:41.602 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:41.630 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : util-linux-user-2.32.1-43.el8.x86_64 132/162 2024-04-06T13:56:41.831 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : xfsprogs-devel-5.0.0-12.el8.x86_64 133/162 2024-04-06T13:56:41.850 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : util-linux-user-2.32.1-43.el8.x86_64 132/162 2024-04-06T13:56:41.982 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 134/162 2024-04-06T13:56:42.061 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : xfsprogs-devel-5.0.0-12.el8.x86_64 133/162 2024-04-06T13:56:42.281 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 134/162 2024-04-06T13:56:44.259 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:44.536 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:44.576 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:44.713 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:45.009 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : elfutils-libs-0.189-3.el8.x86_64 136/162 2024-04-06T13:56:45.030 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:45.070 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:45.240 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 125/162 2024-04-06T13:56:45.376 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : iproute-6.2.0-5.el8_9.x86_64 137/162 2024-04-06T13:56:45.489 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : elfutils-libs-0.189-3.el8.x86_64 136/162 2024-04-06T13:56:45.626 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : xfsdump-3.1.8-7.el8_9.x86_64 138/162 2024-04-06T13:56:45.708 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 126/162 2024-04-06T13:56:45.856 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 139/162 2024-04-06T13:56:45.895 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : iproute-6.2.0-5.el8_9.x86_64 137/162 2024-04-06T13:56:46.047 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 140/162 2024-04-06T13:56:46.051 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 127/162 2024-04-06T13:56:46.142 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : xfsdump-3.1.8-7.el8_9.x86_64 138/162 2024-04-06T13:56:46.241 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:46.314 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 139/162 2024-04-06T13:56:46.360 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 141/162 2024-04-06T13:56:46.546 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 140/162 2024-04-06T13:56:46.624 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : openssl-1:1.1.1k-12.el8_9.x86_64 142/162 2024-04-06T13:56:46.819 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:46.827 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:46.842 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 141/162 2024-04-06T13:56:46.910 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 128/162 2024-04-06T13:56:46.987 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 144/162 2024-04-06T13:56:47.123 INFO:teuthology.orchestra.run.smithi134.stdout: Upgrading : openssl-1:1.1.1k-12.el8_9.x86_64 142/162 2024-04-06T13:56:47.212 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : openssl-devel-1:1.1.1k-12.el8_9.x86_64 129/162 2024-04-06T13:56:47.230 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:47.306 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:47.321 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:47.332 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:47.482 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : elfutils-libelf-devel-0.189-3.el8.x86_64 130/162 2024-04-06T13:56:47.527 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:47.539 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:47.551 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 144/162 2024-04-06T13:56:47.696 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:47.746 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : elfutils-libs-0.186-1.el8.x86_64 147/162 2024-04-06T13:56:47.753 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:47.836 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:47.923 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : openssl-1:1.1.1k-6.el8_5.x86_64 148/162 2024-04-06T13:56:47.969 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : util-linux-user-2.32.1-43.el8.x86_64 132/162 2024-04-06T13:56:48.079 INFO:teuthology.orchestra.run.smithi134.stdout: Installing : bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:48.187 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : xfsprogs-devel-5.0.0-12.el8.x86_64 133/162 2024-04-06T13:56:48.187 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:48.187 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : xfsdump-3.1.8-4.el8.x86_64 149/162 2024-04-06T13:56:48.187 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:48.314 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : elfutils-libs-0.186-1.el8.x86_64 147/162 2024-04-06T13:56:48.350 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 134/162 2024-04-06T13:56:48.404 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:48.405 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:48.531 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : openssl-1:1.1.1k-6.el8_5.x86_64 148/162 2024-04-06T13:56:48.650 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:48.650 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:48.699 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : xfsdump-3.1.8-4.el8.x86_64 149/162 2024-04-06T13:56:48.699 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:48.877 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:49.007 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:49.007 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:49.098 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 153/162 2024-04-06T13:56:49.266 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:49.266 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:49.266 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : util-linux-user-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:49.486 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : util-linux-2.32.1-35.el8.x86_64 155/162 2024-04-06T13:56:49.487 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:49.510 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:49.716 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:49.716 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:49.730 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 153/162 2024-04-06T13:56:49.889 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : util-linux-user-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:49.932 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:49.932 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:50.131 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : util-linux-2.32.1-35.el8.x86_64 155/162 2024-04-06T13:56:50.131 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:50.182 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:50.182 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:50.374 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:50.374 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:50.389 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:50.390 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:50.575 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:50.624 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:50.673 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:50.816 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : elfutils-libelf-0.186-1.el8.x86_64 161/162 2024-04-06T13:56:50.816 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:50.858 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:50.889 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:50.890 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:50.897 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: postgresql-server-10.23-4.module+el8.9.0+21317+d 135/162 2024-04-06T13:56:51.164 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:51.164 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:51.378 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : elfutils-libs-0.189-3.el8.x86_64 136/162 2024-04-06T13:56:51.414 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:51.573 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : elfutils-libelf-0.186-1.el8.x86_64 161/162 2024-04-06T13:56:51.573 INFO:teuthology.orchestra.run.smithi134.stdout: Cleanup : openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:51.746 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:51.746 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 1/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 3/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 4/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 5/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 6/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 7/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 8/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 13/162 2024-04-06T13:56:51.747 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 14/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 15/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 16/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 17/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 18/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 20/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 21/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 22/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 23/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 24/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 25/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 26/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 27/162 2024-04-06T13:56:51.748 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 28/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 29/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 30/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 31/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 32/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 33/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 34/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 35/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 36/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 37/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 38/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/162 2024-04-06T13:56:51.749 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/162 2024-04-06T13:56:51.750 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 60/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/162 2024-04-06T13:56:51.751 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 72/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 73/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 74/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 75/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 76/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 77/162 2024-04-06T13:56:51.752 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : bison-3.0.4-10.el8.x86_64 78/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : flex-2.6.1-9.el8.x86_64 79/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 80/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 81/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 82/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 83/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 84/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 85/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 86/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 87/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 88/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 89/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 90/162 2024-04-06T13:56:51.753 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openblas-threads-0.3.15-6.el8.x86_64 91/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 92/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openblas-0.3.15-6.el8.x86_64 93/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : automake-1.16.1-8.el8.noarch 94/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libpq-13.11-1.el8.x86_64 95/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : jq-1.6-7.el8.x86_64 96/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 97/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 98/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : oniguruma-6.8.2-2.1.el8_9.x86_64 99/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 100/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 101/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : postgresql-server-10.23-4.module+el8.9.0+21317+d 102/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 103/162 2024-04-06T13:56:51.754 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 104/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 105/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 106/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 107/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 108/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libzstd-devel-1.4.4-1.el8.x86_64 109/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 110/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 111/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 112/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 113/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 114/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 115/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 116/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 117/162 2024-04-06T13:56:51.755 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 118/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 119/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 120/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libuuid-devel-2.32.1-43.el8.x86_64 121/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 122/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xfsprogs-devel-5.0.0-12.el8.x86_64 123/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : NetworkManager-initscripts-updown-1:1.40.16-13.e 124/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : elfutils-libelf-devel-0.189-3.el8.x86_64 125/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : krb5-devel-1.18.2-26.el8_9.x86_64 126/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openssl-devel-1:1.1.1k-12.el8_9.x86_64 127/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librabbitmq-0.9.0-5.el8_9.x86_64 128/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cryptography-3.2.1-7.el8_9.x86_64 129/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-packaging-16.8-9.el8.noarch 130/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:51.756 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 132/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 133/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 134/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : elfutils-libelf-0.189-3.el8.x86_64 135/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : elfutils-libelf-0.186-1.el8.x86_64 136/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : elfutils-libs-0.189-3.el8.x86_64 137/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : elfutils-libs-0.186-1.el8.x86_64 138/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libmount-2.32.1-43.el8.x86_64 139/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libmount-2.32.1-35.el8.x86_64 140/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libsmartcols-2.32.1-43.el8.x86_64 141/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libsmartcols-2.32.1-35.el8.x86_64 142/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libuuid-2.32.1-43.el8.x86_64 143/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libuuid-2.32.1-35.el8.x86_64 144/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : util-linux-2.32.1-43.el8.x86_64 145/162 2024-04-06T13:56:51.757 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : util-linux-2.32.1-35.el8.x86_64 146/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : util-linux-user-2.32.1-43.el8.x86_64 147/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : util-linux-user-2.32.1-35.el8.x86_64 148/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : iproute-6.2.0-5.el8_9.x86_64 149/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 150/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libblkid-2.32.1-43.el8.x86_64 151/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libblkid-2.32.1-35.el8.x86_64 152/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libfdisk-2.32.1-43.el8.x86_64 153/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libfdisk-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openssl-1:1.1.1k-12.el8_9.x86_64 155/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openssl-1:1.1.1k-6.el8_5.x86_64 156/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openssl-libs-1:1.1.1k-12.el8_9.x86_64 157/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openssl-libs-1:1.1.1k-6.el8_5.x86_64 158/162 2024-04-06T13:56:51.758 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xfsdump-3.1.8-7.el8_9.x86_64 159/162 2024-04-06T13:56:51.759 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xfsdump-3.1.8-4.el8.x86_64 160/162 2024-04-06T13:56:51.759 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xfsprogs-5.0.0-12.el8.x86_64 161/162 2024-04-06T13:56:51.797 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : iproute-6.2.0-5.el8_9.x86_64 137/162 2024-04-06T13:56:52.028 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : xfsdump-3.1.8-7.el8_9.x86_64 138/162 2024-04-06T13:56:52.232 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 139/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Running scriptlet: openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 1/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 3/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 4/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 5/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 6/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 7/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 8/162 2024-04-06T13:56:52.490 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 13/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 14/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 15/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 16/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 17/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 18/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 20/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 21/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 22/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 23/162 2024-04-06T13:56:52.491 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 24/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 25/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 26/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 27/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 28/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 29/162 2024-04-06T13:56:52.492 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 30/162 2024-04-06T13:56:52.506 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 140/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 31/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 32/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 33/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 34/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 35/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 36/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 37/162 2024-04-06T13:56:52.507 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 38/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/162 2024-04-06T13:56:52.508 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 60/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/162 2024-04-06T13:56:52.509 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 72/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 73/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 74/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 75/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 76/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 77/162 2024-04-06T13:56:52.510 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : bison-3.0.4-10.el8.x86_64 78/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : flex-2.6.1-9.el8.x86_64 79/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 80/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 81/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 82/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 83/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 84/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 85/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 86/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 87/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 88/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 89/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 90/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openblas-threads-0.3.15-6.el8.x86_64 91/162 2024-04-06T13:56:52.511 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 92/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openblas-0.3.15-6.el8.x86_64 93/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : automake-1.16.1-8.el8.noarch 94/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libpq-13.11-1.el8.x86_64 95/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : jq-1.6-7.el8.x86_64 96/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 97/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 98/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : oniguruma-6.8.2-2.1.el8_9.x86_64 99/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 100/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 101/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : postgresql-server-10.23-4.module+el8.9.0+21317+d 102/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-packaging-16.8-9.el8.noarch 103/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 104/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 105/162 2024-04-06T13:56:52.512 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 106/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 107/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 108/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 109/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libzstd-devel-1.4.4-1.el8.x86_64 110/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 111/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 112/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 113/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 114/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 115/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 116/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 117/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 118/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 119/162 2024-04-06T13:56:52.513 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 120/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 121/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libuuid-devel-2.32.1-43.el8.x86_64 122/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 123/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xfsprogs-devel-5.0.0-12.el8.x86_64 124/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : NetworkManager-initscripts-updown-1:1.40.16-13.e 125/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : elfutils-libelf-devel-0.189-3.el8.x86_64 126/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : krb5-devel-1.18.2-26.el8_9.x86_64 127/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openssl-devel-1:1.1.1k-12.el8_9.x86_64 128/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librabbitmq-0.9.0-5.el8_9.x86_64 129/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : python3-cryptography-3.2.1-7.el8_9.x86_64 130/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 132/162 2024-04-06T13:56:52.514 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 133/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 134/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : elfutils-libelf-0.189-3.el8.x86_64 135/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : elfutils-libelf-0.186-1.el8.x86_64 136/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : elfutils-libs-0.189-3.el8.x86_64 137/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : elfutils-libs-0.186-1.el8.x86_64 138/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libmount-2.32.1-43.el8.x86_64 139/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libmount-2.32.1-35.el8.x86_64 140/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libsmartcols-2.32.1-43.el8.x86_64 141/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libsmartcols-2.32.1-35.el8.x86_64 142/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libuuid-2.32.1-43.el8.x86_64 143/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libuuid-2.32.1-35.el8.x86_64 144/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : util-linux-2.32.1-43.el8.x86_64 145/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : util-linux-2.32.1-35.el8.x86_64 146/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : util-linux-user-2.32.1-43.el8.x86_64 147/162 2024-04-06T13:56:52.515 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : util-linux-user-2.32.1-35.el8.x86_64 148/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : iproute-6.2.0-5.el8_9.x86_64 149/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 150/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libblkid-2.32.1-43.el8.x86_64 151/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libblkid-2.32.1-35.el8.x86_64 152/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libfdisk-2.32.1-43.el8.x86_64 153/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : libfdisk-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openssl-1:1.1.1k-12.el8_9.x86_64 155/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openssl-1:1.1.1k-6.el8_5.x86_64 156/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openssl-libs-1:1.1.1k-12.el8_9.x86_64 157/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : openssl-libs-1:1.1.1k-6.el8_5.x86_64 158/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xfsdump-3.1.8-7.el8_9.x86_64 159/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xfsdump-3.1.8-4.el8.x86_64 160/162 2024-04-06T13:56:52.516 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xfsprogs-5.0.0-12.el8.x86_64 161/162 2024-04-06T13:56:52.827 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 141/162 2024-04-06T13:56:53.109 INFO:teuthology.orchestra.run.smithi090.stdout: Upgrading : openssl-1:1.1.1k-12.el8_9.x86_64 142/162 2024-04-06T13:56:53.266 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:53.277 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.40.16-13.e 143/162 2024-04-06T13:56:53.436 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 144/162 2024-04-06T13:56:53.698 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:53.778 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 145/162 2024-04-06T13:56:53.974 INFO:teuthology.orchestra.run.smithi090.stdout: Installing : bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:53.988 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 146/162 2024-04-06T13:56:54.180 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : elfutils-libs-0.186-1.el8.x86_64 147/162 2024-04-06T13:56:54.367 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : openssl-1:1.1.1k-6.el8_5.x86_64 148/162 2024-04-06T13:56:54.536 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : xfsdump-3.1.8-4.el8.x86_64 149/162 2024-04-06T13:56:54.536 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:54.794 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: xfsprogs-5.0.0-10.el8.x86_64 150/162 2024-04-06T13:56:54.794 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:55.053 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 151/162 2024-04-06T13:56:55.053 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:55.305 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 152/162 2024-04-06T13:56:55.559 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 153/162 2024-04-06T13:56:55.708 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : util-linux-user-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:55.917 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : util-linux-2.32.1-35.el8.x86_64 155/162 2024-04-06T13:56:55.917 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:56.185 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libfdisk-2.32.1-35.el8.x86_64 156/162 2024-04-06T13:56:56.185 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:56.410 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libmount-2.32.1-35.el8.x86_64 157/162 2024-04-06T13:56:56.410 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:56.643 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libblkid-2.32.1-35.el8.x86_64 158/162 2024-04-06T13:56:56.643 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:56.892 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libuuid-2.32.1-35.el8.x86_64 159/162 2024-04-06T13:56:56.893 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:57.109 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: libsmartcols-2.32.1-35.el8.x86_64 160/162 2024-04-06T13:56:57.276 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : elfutils-libelf-0.186-1.el8.x86_64 161/162 2024-04-06T13:56:57.276 INFO:teuthology.orchestra.run.smithi090.stdout: Cleanup : openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:58.169 INFO:teuthology.orchestra.run.smithi090.stdout: Running scriptlet: openssl-libs-1:1.1.1k-6.el8_5.x86_64 162/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-2:18.2.2-1086.g052adcad.el8.x86_64 1/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 3/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 4/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-immutable-object-cache-2:18.2.2-1086.g052ad 5/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 6/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 7/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 8/162 2024-04-06T13:56:58.170 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 9/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 10/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 11/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 12/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 13/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_ 14/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 15/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 16/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados-devel-2:18.2.2-1086.g052adcad.el8.x86_6 17/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86 18/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 19/162 2024-04-06T13:56:58.171 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-ceph-argparse-2:18.2.2-1086.g052adcad.el 20/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-ceph-common-2:18.2.2-1086.g052adcad.el8. 21/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_6 22/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 23/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 24/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 25/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 26/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 27/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 28/162 2024-04-06T13:56:58.172 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-grafana-dashboards-2:18.2.2-1086.g052adcad. 29/162 2024-04-06T13:56:58.186 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noa 30/162 2024-04-06T13:56:58.186 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.n 31/162 2024-04-06T13:56:58.186 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.2-1086.g052 32/162 2024-04-06T13:56:58.186 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el 33/162 2024-04-06T13:56:58.186 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 34/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.e 35/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 36/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : cephadm-2:18.2.2-1086.g052adcad.el8.noarch 37/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 38/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/162 2024-04-06T13:56:58.187 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 60/162 2024-04-06T13:56:58.188 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-packaging-16.8-9.el8.noarch 72/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 73/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 74/162 2024-04-06T13:56:58.189 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 75/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 76/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 77/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 78/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : bison-3.0.4-10.el8.x86_64 79/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : flex-2.6.1-9.el8.x86_64 80/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 81/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 82/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 83/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 84/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 85/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 86/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 87/162 2024-04-06T13:56:58.190 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 88/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 89/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 90/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 91/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openblas-threads-0.3.15-6.el8.x86_64 92/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 93/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openblas-0.3.15-6.el8.x86_64 94/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : automake-1.16.1-8.el8.noarch 95/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libpq-13.11-1.el8.x86_64 96/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : jq-1.6-7.el8.x86_64 97/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 98/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c3 99/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : oniguruma-6.8.2-2.1.el8_9.x86_64 100/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a 101/162 2024-04-06T13:56:58.191 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : postgresql-contrib-10.23-4.module+el8.9.0+21317+ 102/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : postgresql-server-10.23-4.module+el8.9.0+21317+d 103/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 104/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 105/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 106/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 107/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 108/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 109/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libzstd-devel-1.4.4-1.el8.x86_64 110/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 111/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 112/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 113/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 114/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 115/162 2024-04-06T13:56:58.192 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 116/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 117/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 118/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 119/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 120/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 121/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libuuid-devel-2.32.1-43.el8.x86_64 122/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 123/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xfsprogs-devel-5.0.0-12.el8.x86_64 124/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : NetworkManager-initscripts-updown-1:1.40.16-13.e 125/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : elfutils-libelf-devel-0.189-3.el8.x86_64 126/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : krb5-devel-1.18.2-26.el8_9.x86_64 127/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openssl-devel-1:1.1.1k-12.el8_9.x86_64 128/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librabbitmq-0.9.0-5.el8_9.x86_64 129/162 2024-04-06T13:56:58.193 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : python3-cryptography-3.2.1-7.el8_9.x86_64 130/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados2-2:18.2.2-1086.g052adcad.el8.x86_64 131/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 132/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 133/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 134/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : elfutils-libelf-0.189-3.el8.x86_64 135/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : elfutils-libelf-0.186-1.el8.x86_64 136/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : elfutils-libs-0.189-3.el8.x86_64 137/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : elfutils-libs-0.186-1.el8.x86_64 138/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libmount-2.32.1-43.el8.x86_64 139/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libmount-2.32.1-35.el8.x86_64 140/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libsmartcols-2.32.1-43.el8.x86_64 141/162 2024-04-06T13:56:58.194 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libsmartcols-2.32.1-35.el8.x86_64 142/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libuuid-2.32.1-43.el8.x86_64 143/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libuuid-2.32.1-35.el8.x86_64 144/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : util-linux-2.32.1-43.el8.x86_64 145/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : util-linux-2.32.1-35.el8.x86_64 146/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : util-linux-user-2.32.1-43.el8.x86_64 147/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : util-linux-user-2.32.1-35.el8.x86_64 148/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : iproute-6.2.0-5.el8_9.x86_64 149/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 150/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libblkid-2.32.1-43.el8.x86_64 151/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libblkid-2.32.1-35.el8.x86_64 152/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libfdisk-2.32.1-43.el8.x86_64 153/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : libfdisk-2.32.1-35.el8.x86_64 154/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openssl-1:1.1.1k-12.el8_9.x86_64 155/162 2024-04-06T13:56:58.195 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openssl-1:1.1.1k-6.el8_5.x86_64 156/162 2024-04-06T13:56:58.196 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openssl-libs-1:1.1.1k-12.el8_9.x86_64 157/162 2024-04-06T13:56:58.196 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : openssl-libs-1:1.1.1k-6.el8_5.x86_64 158/162 2024-04-06T13:56:58.196 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xfsdump-3.1.8-7.el8_9.x86_64 159/162 2024-04-06T13:56:58.196 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xfsdump-3.1.8-4.el8.x86_64 160/162 2024-04-06T13:56:58.196 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xfsprogs-5.0.0-12.el8.x86_64 161/162 2024-04-06T13:57:04.757 INFO:teuthology.orchestra.run.smithi134.stdout: Verifying : xfsprogs-5.0.0-10.el8.x86_64 162/162 2024-04-06T13:57:04.757 INFO:teuthology.orchestra.run.smithi134.stdout:Installed products updated. 2024-04-06T13:57:05.119 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xfsprogs-5.0.0-10.el8.x86_64 162/162 2024-04-06T13:57:05.119 INFO:teuthology.orchestra.run.smithi105.stdout:Installed products updated. 2024-04-06T13:57:05.979 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout:Upgraded: 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libelf-0.189-3.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libs-0.189-3.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: iproute-6.2.0-5.el8_9.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: libblkid-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: libfdisk-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: libmount-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: libsmartcols-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: libuuid-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: openssl-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:05.980 INFO:teuthology.orchestra.run.smithi134.stdout: openssl-libs-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: util-linux-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: util-linux-user-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: xfsdump-3.1.8-7.el8_9.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: xfsprogs-5.0.0-12.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout:Installed: 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: NetworkManager-initscripts-updown-1:1.40.16-13.el8_9.noarch 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: automake-1.16.1-8.el8.noarch 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: bison-3.0.4-10.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-grafana-dashboards-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.981 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-immutable-object-cache-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.982 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: elfutils-libelf-devel-0.189-3.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: flex-2.6.1-9.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: fuse-2.9.7-17.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: jq-1.6-7.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: krb5-devel-1.18.2-26.el8_9.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: libaio-devel-0.3.112-1.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.983 INFO:teuthology.orchestra.run.smithi134.stdout: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libgfortran-8.5.0-20.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: liboath-2.6.2-3.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libpq-13.11-1.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libquadmath-8.5.0-20.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: librabbitmq-0.9.0-5.el8_9.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: librados-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: librdkafka-0.11.4-3.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libselinux-devel-2.9-5.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libsepol-devel-2.9-3.el8.x86_64 2024-04-06T13:57:05.984 INFO:teuthology.orchestra.run.smithi134.stdout: libtool-2.4.6-25.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: libunwind-1.3.1-3.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: libuuid-devel-2.32.1-43.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: libverto-devel-0.3.0-5.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: libzstd-devel-1.4.4-1.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: lttng-ust-2.8.1-11.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: oniguruma-6.8.2-2.1.el8_9.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-0.3.15-6.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: openblas-threads-0.3.15-6.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: openssl-devel-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-devel-10.32-2.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-utf16-10.32-2.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: pcre2-utf32-10.32-2.el8.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql-contrib-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:05.985 INFO:teuthology.orchestra.run.smithi134.stdout: postgresql-server-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-babel-2.5.1-7.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cachetools-3.1.1-4.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-argparse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-certifi-2018.10.15-7.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cffi-1.11.5-6.el8.x86_64 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cheroot-8.5.2-1.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2024-04-06T13:57:05.986 INFO:teuthology.orchestra.run.smithi134.stdout: python3-cryptography-3.2.1-7.el8_9.x86_64 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-6.2-6.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jinja2-2.10.1-3.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpatch-1.21-2.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jsonpointer-1.10-11.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-jwt-1.6.1-2.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-logutils-0.3.5-11.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-mako-1.0.6-14.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-markupsafe-0.23-19.el8.x86_64 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2024-04-06T13:57:05.987 INFO:teuthology.orchestra.run.smithi134.stdout: python3-natsort-7.1.1-2.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-packaging-16.8-9.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pecan-1.3.2-9.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-portend-2.6-1.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-prettytable-0.7.2-14.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pycparser-2.14-14.el8.noarch 2024-04-06T13:57:05.988 INFO:teuthology.orchestra.run.smithi134.stdout: python3-pytz-2017.2-11.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-repoze-lru-0.7-6.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-routes-2.4.1-12.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-rsa-4.9-2.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c38525.x86_64 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-tempora-1.14.1-5.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-trustme-0.6.0-4.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-waitress-1.4.3-1.el8.noarch 2024-04-06T13:57:05.989 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webob-1.8.5-1.el8.1.noarch 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-webtest-2.0.33-1.el8.noarch 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: socat-1.7.4.1-1.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: thrift-0.13.0-2.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: uuid-1.6.2-43.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: xfsprogs-devel-5.0.0-12.el8.x86_64 2024-04-06T13:57:05.990 INFO:teuthology.orchestra.run.smithi134.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2024-04-06T13:57:05.991 INFO:teuthology.orchestra.run.smithi134.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2024-04-06T13:57:05.991 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:57:05.991 INFO:teuthology.orchestra.run.smithi134.stdout:Complete! 2024-04-06T13:57:06.258 DEBUG:teuthology.parallel:result is None 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout:Upgraded: 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libelf-0.189-3.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libs-0.189-3.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: iproute-6.2.0-5.el8_9.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: libblkid-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: libfdisk-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: libmount-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.727 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: libsmartcols-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: libuuid-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: openssl-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: openssl-libs-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: util-linux-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: util-linux-user-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: xfsdump-3.1.8-7.el8_9.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: xfsprogs-5.0.0-12.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: NetworkManager-initscripts-updown-1:1.40.16-13.el8_9.noarch 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: automake-1.16.1-8.el8.noarch 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: bison-3.0.4-10.el8.x86_64 2024-04-06T13:57:06.728 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-grafana-dashboards-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.729 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.730 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: elfutils-libelf-devel-0.189-3.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: flex-2.6.1-9.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: fuse-2.9.7-17.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: jq-1.6-7.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: krb5-devel-1.18.2-26.el8_9.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: libaio-devel-0.3.112-1.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.731 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libgfortran-8.5.0-20.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: liboath-2.6.2-3.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libpq-13.11-1.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libquadmath-8.5.0-20.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq-0.9.0-5.el8_9.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka-0.11.4-3.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libselinux-devel-2.9-5.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libsepol-devel-2.9-3.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libtool-2.4.6-25.el8.x86_64 2024-04-06T13:57:06.732 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind-1.3.1-3.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: libuuid-devel-2.32.1-43.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: libverto-devel-0.3.0-5.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: libzstd-devel-1.4.4-1.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust-2.8.1-11.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: oniguruma-6.8.2-2.1.el8_9.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-0.3.15-6.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-threads-0.3.15-6.el8.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: openssl-devel-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:06.733 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-devel-10.32-2.el8.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-utf16-10.32-2.el8.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: pcre2-utf32-10.32-2.el8.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql-contrib-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: postgresql-server-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-babel-2.5.1-7.el8.noarch 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cachetools-3.1.1-4.el8.noarch 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.734 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-certifi-2018.10.15-7.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi-1.11.5-6.el8.x86_64 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot-8.5.2-1.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography-3.2.1-7.el8_9.x86_64 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-6.2-6.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jinja2-2.10.1-3.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpatch-1.21-2.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpointer-1.10-11.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt-1.6.1-2.el8.noarch 2024-04-06T13:57:06.735 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils-0.3.5-11.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako-1.0.6-14.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe-0.23-19.el8.x86_64 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-natsort-7.1.1-2.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging-16.8-9.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan-1.3.2-9.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend-2.6-1.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable-0.7.2-14.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2024-04-06T13:57:06.736 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser-2.14-14.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz-2017.2-11.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-repoze-lru-0.7-6.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-routes-2.4.1-12.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rsa-4.9-2.el8.noarch 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c38525.x86_64 2024-04-06T13:57:06.737 INFO:teuthology.orchestra.run.smithi105.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora-1.14.1-5.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-trustme-0.6.0-4.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-waitress-1.4.3-1.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob-1.8.5-1.el8.1.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webtest-2.0.33-1.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:06.738 INFO:teuthology.orchestra.run.smithi105.stdout: socat-1.7.4.1-1.el8.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: thrift-0.13.0-2.el8.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: uuid-1.6.2-43.el8.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: xfsprogs-devel-5.0.0-12.el8.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-04-06T13:57:06.739 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-04-06T13:57:07.234 DEBUG:teuthology.parallel:result is None 2024-04-06T13:57:15.312 INFO:teuthology.orchestra.run.smithi090.stdout: Verifying : xfsprogs-5.0.0-10.el8.x86_64 162/162 2024-04-06T13:57:15.312 INFO:teuthology.orchestra.run.smithi090.stdout:Installed products updated. 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout:Upgraded: 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libelf-0.189-3.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libs-0.189-3.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: iproute-6.2.0-5.el8_9.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: libblkid-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: libfdisk-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: libmount-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: librados2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: librbd1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.905 INFO:teuthology.orchestra.run.smithi090.stdout: libsmartcols-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: libuuid-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: openssl-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: openssl-libs-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: util-linux-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: util-linux-user-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: xfsdump-3.1.8-7.el8_9.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: xfsprogs-5.0.0-12.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout:Installed: 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: NetworkManager-initscripts-updown-1:1.40.16-13.el8_9.noarch 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: automake-1.16.1-8.el8.noarch 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: bison-3.0.4-10.el8.x86_64 2024-04-06T13:57:15.906 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-base-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-grafana-dashboards-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-immutable-object-cache-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mds-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-dashboard-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-diskprediction-local-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-modules-core-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mgr-rook-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-mon-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.907 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-osd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-prometheus-alerts-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-radosgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-selinux-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-test-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: ceph-volume-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: cephadm-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: cephfs-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: cephfs-top-2:18.2.2-1086.g052adcad.el8.noarch 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: elfutils-libelf-devel-0.189-3.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: flex-2.6.1-9.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: fuse-2.9.7-17.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2024-04-06T13:57:15.908 INFO:teuthology.orchestra.run.smithi090.stdout: jq-1.6-7.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: krb5-devel-1.18.2-26.el8_9.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libaio-devel-0.3.112-1.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libcephfs2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libcephsqlite-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libgfortran-8.5.0-20.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: liboath-2.6.2-3.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libpq-13.11-1.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: libquadmath-8.5.0-20.el8.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: librabbitmq-0.9.0-5.el8_9.x86_64 2024-04-06T13:57:15.909 INFO:teuthology.orchestra.run.smithi090.stdout: librados-devel-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libradosstriper1-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: librdkafka-0.11.4-3.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: librgw2-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libselinux-devel-2.9-5.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libsepol-devel-2.9-3.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libtool-2.4.6-25.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libunwind-1.3.1-3.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libuuid-devel-2.32.1-43.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libverto-devel-0.3.0-5.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: libzstd-devel-1.4.4-1.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: lttng-ust-2.8.1-11.el8.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: oniguruma-6.8.2-2.1.el8_9.x86_64 2024-04-06T13:57:15.910 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-0.3.15-6.el8.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: openblas-threads-0.3.15-6.el8.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: openssl-devel-1:1.1.1k-12.el8_9.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-devel-10.32-2.el8.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-utf16-10.32-2.el8.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: pcre2-utf32-10.32-2.el8.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql-contrib-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: postgresql-server-10.23-4.module+el8.9.0+21317+d2c6bf8a.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: python3-babel-2.5.1-7.el8.noarch 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2024-04-06T13:57:15.911 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cachetools-3.1.1-4.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-argparse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-ceph-common-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cephfs-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-certifi-2018.10.15-7.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cffi-1.11.5-6.el8.x86_64 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cheroot-8.5.2-1.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-cryptography-3.2.1-7.el8_9.x86_64 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-6.2-6.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jinja2-2.10.1-3.el8.noarch 2024-04-06T13:57:15.912 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpatch-1.21-2.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jsonpointer-1.10-11.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-jwt-1.6.1-2.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-logutils-0.3.5-11.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-mako-1.0.6-14.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-markupsafe-0.23-19.el8.x86_64 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-natsort-7.1.1-2.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-packaging-16.8-9.el8.noarch 2024-04-06T13:57:15.913 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pecan-1.3.2-9.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-portend-2.6-1.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-prettytable-0.7.2-14.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pycparser-2.14-14.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-pytz-2017.2-11.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rados-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-repoze-lru-0.7-6.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rgw-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-routes-2.4.1-12.el8.noarch 2024-04-06T13:57:15.914 INFO:teuthology.orchestra.run.smithi090.stdout: python3-rsa-4.9-2.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-scipy-1.0.0-21.module+el8.9.0+20976+d3c38525.x86_64 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-tempora-1.14.1-5.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-trustme-0.6.0-4.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-waitress-1.4.3-1.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webob-1.8.5-1.el8.1.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-webtest-2.0.33-1.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-fuse-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.915 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-mirror-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: rbd-nbd-2:18.2.2-1086.g052adcad.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: socat-1.7.4.1-1.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: thrift-0.13.0-2.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: uuid-1.6.2-43.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: xfsprogs-devel-5.0.0-12.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2024-04-06T13:57:15.916 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:57:15.917 INFO:teuthology.orchestra.run.smithi090.stdout:Complete! 2024-04-06T13:57:16.162 DEBUG:teuthology.parallel:result is None 2024-04-06T13:57:16.162 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:16.307 DEBUG:teuthology.orchestra.run.smithi090:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-06T13:57:16.330 INFO:teuthology.orchestra.run.smithi090.stdout:18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.331 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.331 INFO:teuthology.task.install:The correct ceph version 18.2.2-1086.g052adcad is installed. 2024-04-06T13:57:16.333 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:16.632 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-06T13:57:16.662 INFO:teuthology.orchestra.run.smithi105.stdout:18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.662 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.662 INFO:teuthology.task.install:The correct ceph version 18.2.2-1086.g052adcad is installed. 2024-04-06T13:57:16.665 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:16.813 DEBUG:teuthology.orchestra.run.smithi134:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-06T13:57:16.841 INFO:teuthology.orchestra.run.smithi134.stdout:18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.842 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1086.g052adcad.el8 2024-04-06T13:57:16.842 INFO:teuthology.task.install:The correct ceph version 18.2.2-1086.g052adcad is installed. 2024-04-06T13:57:16.844 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-06T13:57:16.844 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:57:16.844 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-06T13:57:16.890 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:57:16.890 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-06T13:57:16.940 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:57:16.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-06T13:57:16.988 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-06T13:57:16.989 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:57:16.989 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/daemon-helper 2024-04-06T13:57:17.018 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-06T13:57:17.086 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:57:17.086 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/daemon-helper 2024-04-06T13:57:17.117 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-06T13:57:17.186 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:57:17.186 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/daemon-helper 2024-04-06T13:57:17.220 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-06T13:57:17.291 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-06T13:57:17.291 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:57:17.291 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-06T13:57:17.319 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-06T13:57:17.387 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:57:17.387 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-06T13:57:17.418 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-06T13:57:17.488 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:57:17.489 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-06T13:57:17.552 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-06T13:57:17.582 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-06T13:57:17.582 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:57:17.582 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/usr/bin/stdin-killer 2024-04-06T13:57:17.610 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-06T13:57:17.678 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:57:17.678 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/stdin-killer 2024-04-06T13:57:17.710 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-06T13:57:17.780 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:57:17.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/usr/bin/stdin-killer 2024-04-06T13:57:17.813 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-06T13:57:17.885 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-06T13:57:17.965 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'max_mds': 3, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'mds_bal_replicate_threshold': 1, 'mds_export_ephemeral_random_max': 0.1, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug cephsqlite': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': 20, 'debug bluestore': 20, 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': 10, 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'slow metadata IO', 'SLOW_OPS', 'slow request'], 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'subvols': {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'}} 2024-04-06T13:57:17.965 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:17.966 INFO:tasks.cephadm:Cluster fsid is 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:57:17.966 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-06T13:57:17.966 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.90', 'mon.b': '172.21.15.105', 'mon.c': '172.21.15.134'} 2024-04-06T13:57:17.966 INFO:tasks.cephadm:First mon is mon.a on smithi090 2024-04-06T13:57:17.966 INFO:tasks.cephadm:First mgr is x 2024-04-06T13:57:17.966 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-06T13:57:17.967 DEBUG:teuthology.orchestra.run.smithi090:> sudo hostname $(hostname -s) 2024-04-06T13:57:17.994 DEBUG:teuthology.orchestra.run.smithi105:> sudo hostname $(hostname -s) 2024-04-06T13:57:18.027 DEBUG:teuthology.orchestra.run.smithi134:> sudo hostname $(hostname -s) 2024-04-06T13:57:18.060 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2024-04-06T13:57:18.061 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:18.392 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/54799/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.2.2-1086-g052adcad', 'node_name': '172.21.4.63+confusa01', 'job_name': 'ceph-dev-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.2.2-1086.g052adcad'}, 'url': 'https://4.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2024-04-03 05:21:39.447031', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'reef', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/', 'archs': ['source', 'x86_64', 'arm64'], 'distro': 'centos'}, {'status': 'ready', 'sha1': '052adcadd0534eddb0d73c5afbc083a84cd9dc46', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/54799/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.2.2-1086-g052adcad', 'node_name': '172.21.5.32+adami02', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.2.2-1086.g052adcad'}, 'url': 'https://3.chacra.ceph.com/r/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2024-04-03 03:23:29.633707', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'reef', 'chacra_url': 'https://3.chacra.ceph.com/repos/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-04-06T13:57:18.732 INFO:tasks.util.chacra:got chacra host 4.chacra.ceph.com, ref reef, sha1 052adcadd0534eddb0d73c5afbc083a84cd9dc46 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F8%2Fx86_64&flavor=default&sha1=052adcadd0534eddb0d73c5afbc083a84cd9dc46 2024-04-06T13:57:18.733 INFO:tasks.cephadm:Discovered cachra url: https://4.chacra.ceph.com/binaries/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/x86_64/flavors/default/cephadm 2024-04-06T13:57:18.734 DEBUG:teuthology.orchestra.run.smithi090:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:19.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-rw-r--. 1 ubuntu ubuntu 703022 Apr 6 13:57 /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:19.184 DEBUG:teuthology.orchestra.run.smithi105:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:19.626 INFO:teuthology.orchestra.run.smithi105.stdout:-rw-rw-r--. 1 ubuntu ubuntu 703022 Apr 6 13:57 /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:19.627 DEBUG:teuthology.orchestra.run.smithi134:> curl --silent -L https://4.chacra.ceph.com/binaries/ceph/reef/052adcadd0534eddb0d73c5afbc083a84cd9dc46/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:20.102 INFO:teuthology.orchestra.run.smithi134.stdout:-rw-rw-r--. 1 ubuntu ubuntu 703022 Apr 6 13:57 /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:20.102 DEBUG:teuthology.orchestra.run.smithi090:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:20.124 DEBUG:teuthology.orchestra.run.smithi105:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:20.145 DEBUG:teuthology.orchestra.run.smithi134:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-06T13:57:20.187 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 on all hosts... 2024-04-06T13:57:20.188 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 pull 2024-04-06T13:57:20.190 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 pull 2024-04-06T13:57:20.192 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 pull 2024-04-06T13:57:21.160 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46... 2024-04-06T13:57:21.415 INFO:teuthology.orchestra.run.smithi134.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46... 2024-04-06T13:57:22.361 INFO:teuthology.orchestra.run.smithi090.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46... 2024-04-06T13:57:49.731 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-04-06T13:57:49.731 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_version": "ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable)", 2024-04-06T13:57:49.731 INFO:teuthology.orchestra.run.smithi134.stdout: "image_id": "87261d5ece5af65eae1ff95ff5040de5a8b6f7cf2523cbab84464d2e909acf38", 2024-04-06T13:57:49.731 INFO:teuthology.orchestra.run.smithi134.stdout: "repo_digests": [ 2024-04-06T13:57:49.731 INFO:teuthology.orchestra.run.smithi134.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:45cbf7e6d38f79ab874661480485370918610a7e9a2761c03c5b2e736405702e" 2024-04-06T13:57:49.732 INFO:teuthology.orchestra.run.smithi134.stdout: ] 2024-04-06T13:57:49.732 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph_version": "ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable)", 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout: "image_id": "87261d5ece5af65eae1ff95ff5040de5a8b6f7cf2523cbab84464d2e909acf38", 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout: "repo_digests": [ 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:45cbf7e6d38f79ab874661480485370918610a7e9a2761c03c5b2e736405702e" 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T13:58:00.443 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T13:58:04.778 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_version": "ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable)", 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout: "image_id": "87261d5ece5af65eae1ff95ff5040de5a8b6f7cf2523cbab84464d2e909acf38", 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout: "repo_digests": [ 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:45cbf7e6d38f79ab874661480485370918610a7e9a2761c03c5b2e736405702e" 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-04-06T13:58:04.779 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-04-06T13:58:04.799 DEBUG:teuthology.orchestra.run.smithi090:> sudo mkdir -p /etc/ceph 2024-04-06T13:58:04.837 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph 2024-04-06T13:58:04.869 DEBUG:teuthology.orchestra.run.smithi134:> sudo mkdir -p /etc/ceph 2024-04-06T13:58:04.907 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 777 /etc/ceph 2024-04-06T13:58:04.939 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /etc/ceph 2024-04-06T13:58:04.967 DEBUG:teuthology.orchestra.run.smithi134:> sudo chmod 777 /etc/ceph 2024-04-06T13:58:04.998 INFO:tasks.cephadm:Writing seed config... 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [client] debug client = 20 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [client] debug ms = 1 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2024-04-06T13:58:04.999 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] debug mds = 20 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] debug ms = 1 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds_bal_replicate_threshold = 1 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] mds_export_ephemeral_random_max = 0.1 2024-04-06T13:58:05.000 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mgr] debug cephsqlite = 20 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [osd] bluestore compression mode = aggressive 2024-04-06T13:58:05.001 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] debug bluefs = 20 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] debug bluestore = 20 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] debug rocksdb = 10 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-06T13:58:05.002 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10000 2024-04-06T13:58:05.003 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:58:05.003 DEBUG:teuthology.orchestra.run.smithi090:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-06T13:58:05.027 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 949dca1a-f41d-11ee-b649-cb9ed24678a4 ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bluestore block size = 96636764160 bluestore compression mode = aggressive bluestore fsck on mount = True debug bluefs = 20 debug bluestore = 20 debug ms = 1 debug osd = 20 debug rocksdb = 10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10000 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug cephsqlite = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True mds_bal_replicate_threshold = 1 mds_export_ephemeral_random_max = 0.1 osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2024-04-06T13:58:05.028 DEBUG:teuthology.orchestra.run.smithi090:mon.a> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service 2024-04-06T13:58:05.070 DEBUG:teuthology.orchestra.run.smithi090:mgr.x> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x.service 2024-04-06T13:58:05.101 INFO:journalctl@ceph.mon.a.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T13:58:05.102 INFO:tasks.cephadm:Bootstrapping... 2024-04-06T13:58:05.102 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 -v bootstrap --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.90 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-06T13:58:05.131 INFO:journalctl@ceph.mgr.x.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T13:58:05.267 INFO:teuthology.orchestra.run.smithi090.stdout:-------------------------------------------------------------------------------- 2024-04-06T13:58:05.268 INFO:teuthology.orchestra.run.smithi090.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46', '-v', 'bootstrap', '--fsid', '949dca1a-f41d-11ee-b649-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.90', '--skip-admin-label'] 2024-04-06T13:58:05.300 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.6.1 2024-04-06T13:58:05.300 INFO:teuthology.orchestra.run.smithi090.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-06T13:58:05.301 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying podman|docker is present... 2024-04-06T13:58:05.333 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.6.1 2024-04-06T13:58:05.333 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying lvm2 is present... 2024-04-06T13:58:05.333 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying time synchronization is in place... 2024-04-06T13:58:05.340 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-06T13:58:05.340 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-06T13:58:05.345 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-06T13:58:05.346 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:58:05.352 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout enabled 2024-04-06T13:58:05.357 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout active 2024-04-06T13:58:05.357 INFO:teuthology.orchestra.run.smithi090.stdout:Unit chronyd.service is enabled and running 2024-04-06T13:58:05.357 INFO:teuthology.orchestra.run.smithi090.stdout:Repeating the final host check... 2024-04-06T13:58:05.388 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 4.6.1 2024-04-06T13:58:05.389 INFO:teuthology.orchestra.run.smithi090.stdout:podman (/bin/podman) version 4.6.1 is present 2024-04-06T13:58:05.389 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl is present 2024-04-06T13:58:05.389 INFO:teuthology.orchestra.run.smithi090.stdout:lvcreate is present 2024-04-06T13:58:05.394 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-06T13:58:05.394 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-06T13:58:05.399 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-06T13:58:05.400 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:58:05.405 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout enabled 2024-04-06T13:58:05.412 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout active 2024-04-06T13:58:05.412 INFO:teuthology.orchestra.run.smithi090.stdout:Unit chronyd.service is enabled and running 2024-04-06T13:58:05.412 INFO:teuthology.orchestra.run.smithi090.stdout:Host looks OK 2024-04-06T13:58:05.412 INFO:teuthology.orchestra.run.smithi090.stdout:Cluster fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:05.413 INFO:teuthology.orchestra.run.smithi090.stdout:Acquiring lock 139633221515752 on /run/cephadm/949dca1a-f41d-11ee-b649-cb9ed24678a4.lock 2024-04-06T13:58:05.413 INFO:teuthology.orchestra.run.smithi090.stdout:Lock 139633221515752 acquired on /run/cephadm/949dca1a-f41d-11ee-b649-cb9ed24678a4.lock 2024-04-06T13:58:05.413 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying IP 172.21.15.90 port 3300 ... 2024-04-06T13:58:05.413 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying IP 172.21.15.90 port 6789 ... 2024-04-06T13:58:05.414 INFO:teuthology.orchestra.run.smithi090.stdout:Base mon IP(s) is [172.21.15.90:3300, 172.21.15.90:6789], mon addrv is [v2:172.21.15.90:3300,v1:172.21.15.90:6789] 2024-04-06T13:58:05.416 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp metric 100 2024-04-06T13:58:05.416 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.90 metric 100 2024-04-06T13:58:05.419 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-06T13:58:05.419 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-06T13:58:05.419 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-06T13:58:05.421 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout inet6 fe80::aac0:3b83:4c6e:ef49/64 scope link noprefixroute 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-06T13:58:05.422 INFO:teuthology.orchestra.run.smithi090.stdout:Mon IP `172.21.15.90` is in CIDR network `172.21.0.0/20` 2024-04-06T13:58:05.423 INFO:teuthology.orchestra.run.smithi090.stdout:Mon IP `172.21.15.90` is in CIDR network `172.21.0.0/20` 2024-04-06T13:58:05.423 INFO:teuthology.orchestra.run.smithi090.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-06T13:58:05.423 INFO:teuthology.orchestra.run.smithi090.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-06T13:58:05.424 INFO:teuthology.orchestra.run.smithi090.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46... 2024-04-06T13:58:05.762 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stdout 87261d5ece5af65eae1ff95ff5040de5a8b6f7cf2523cbab84464d2e909acf38 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46... 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Getting image source signatures 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying blob sha256:139115ca2869d631a19fdec2fcaf195cebfacedb8a3727129ccdf79196f7f90a 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying blob sha256:77070339c81bcb5c53bbbf3f31a47f7187e5393cb0e096452852a31192d97570 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Copying config sha256:87261d5ece5af65eae1ff95ff5040de5a8b6f7cf2523cbab84464d2e909acf38 2024-04-06T13:58:05.763 INFO:teuthology.orchestra.run.smithi090.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-06T13:58:06.430 INFO:teuthology.orchestra.run.smithi090.stdout:ceph: stdout ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable) 2024-04-06T13:58:06.430 INFO:teuthology.orchestra.run.smithi090.stdout:Ceph version: ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable) 2024-04-06T13:58:06.430 INFO:teuthology.orchestra.run.smithi090.stdout:Extracting ceph user uid/gid from container image... 2024-04-06T13:58:07.043 INFO:teuthology.orchestra.run.smithi090.stdout:stat: stdout 167 167 2024-04-06T13:58:07.044 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial keys... 2024-04-06T13:58:08.151 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQDvVBFmSbkrEhAALcFt74F/AJhlM4Tauw74Rw== 2024-04-06T13:58:10.477 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQDxVBFmC0ZYDBAA1i2VISCAxlSVPfyy50skcw== 2024-04-06T13:58:11.094 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph-authtool: stdout AQDyVBFmX/+EKxAAM6AoahPVsZtR3qzku9bc+A== 2024-04-06T13:58:11.094 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial monmap... 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:monmaptool for a [v2:172.21.15.90:3300,v1:172.21.15.90:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:setting min_mon_release = pacific 2024-04-06T13:58:11.684 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: set fsid to 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:11.685 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-06T13:58:11.685 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:58:11.685 INFO:teuthology.orchestra.run.smithi090.stdout:Creating mon... 2024-04-06T13:58:12.311 INFO:teuthology.orchestra.run.smithi090.stdout:create mon.a on 2024-04-06T13:58:12.434 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-04-06T13:58:12.526 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-06T13:58:12.657 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target. 2024-04-06T13:58:12.657 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target. 2024-04-06T13:58:12.826 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a 2024-04-06T13:58:12.826 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to reset failed state of unit ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service: Unit ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service not loaded. 2024-04-06T13:58:12.918 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target.wants/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@.service. 2024-04-06T13:58:13.218 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:12 smithi090 systemd[1]: Starting Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:58:13.299 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-06T13:58:13.299 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-06T13:58:13.306 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-06T13:58:13.306 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:58:13.306 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-06T13:58:13.306 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to enable service . firewalld.service is not available 2024-04-06T13:58:13.307 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mon to start... 2024-04-06T13:58:13.307 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mon... 2024-04-06T13:58:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:13 smithi090 bash[42899]: 4bf0ab823090f1f02477e1bc3daf4e4b5581aba807030cc16e7064c7defb1c1c 2024-04-06T13:58:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:13 smithi090 systemd[1]: Started Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:58:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:13 smithi090 ceph-mon[42936]: mkfs 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:13 smithi090 ceph-mon[42936]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout cluster: 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout id: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout services: 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.450202s) 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:14.152 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout data: 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout pgs: 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.693+0000 7f61def0d700 1 Processor -- start 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.693+0000 7f61def0d700 1 -- start start 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.693+0000 7f61def0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.693+0000 7f61def0d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d8105f00 con 0x7f61d8105510 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.694+0000 7f61dcca9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:14.153 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.694+0000 7f61dcca9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45958/0 (socket says 172.21.15.90:45958) 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.694+0000 7f61dcca9700 1 -- 172.21.15.90:0/3349412735 learned_addr learned my addr 172.21.15.90:0/3349412735 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.694+0000 7f61dcca9700 1 -- 172.21.15.90:0/3349412735 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61d8106710 con 0x7f61d8105510 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.694+0000 7f61dcca9700 1 --2- 172.21.15.90:0/3349412735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f61cc0098a0 tx=0x7f61cc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0dfbd537e922d70 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61d77fe700 1 -- 172.21.15.90:0/3349412735 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f61cc004160 con 0x7f61d8105510 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61d77fe700 1 -- 172.21.15.90:0/3349412735 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f61cc0042c0 con 0x7f61d8105510 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61d77fe700 1 -- 172.21.15.90:0/3349412735 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f61cc0045a0 con 0x7f61d8105510 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 -- 172.21.15.90:0/3349412735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 msgr2=0x7f61d8105930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 --2- 172.21.15.90:0/3349412735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f61cc0098a0 tx=0x7f61cc009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 -- 172.21.15.90:0/3349412735 shutdown_connections 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 --2- 172.21.15.90:0/3349412735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d8105930 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:14.154 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 -- 172.21.15.90:0/3349412735 >> 172.21.15.90:0/3349412735 conn(0x7f61d8100a90 msgr2=0x7f61d8102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 -- 172.21.15.90:0/3349412735 shutdown_connections 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.695+0000 7f61def0d700 1 -- 172.21.15.90:0/3349412735 wait complete. 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61def0d700 1 Processor -- start 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61def0d700 1 -- start start 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61def0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61def0d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d8079bf0 con 0x7f61d8105510 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61dcca9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61dcca9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45964/0 (socket says 172.21.15.90:45964) 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61dcca9700 1 -- 172.21.15.90:0/3182922742 learned_addr learned my addr 172.21.15.90:0/3182922742 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61dcca9700 1 -- 172.21.15.90:0/3182922742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61cc009550 con 0x7f61d8105510 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.696+0000 7f61dcca9700 1 --2- 172.21.15.90:0/3182922742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f61cc004000 tx=0x7f61cc004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:14.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f61cc011630 con 0x7f61d8105510 2024-04-06T13:58:14.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f61cc011790 con 0x7f61d8105510 2024-04-06T13:58:14.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f61cc011a30 con 0x7f61d8105510 2024-04-06T13:58:14.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61d8079df0 con 0x7f61d8105510 2024-04-06T13:58:14.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61d819cf70 con 0x7f61d8105510 2024-04-06T13:58:14.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f61cc022070 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.697+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f61cc01c450 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.698+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61d8192780 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.700+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f61cc01b450 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.769+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "status"} v 0) v1 -- 0x7f61d8106a80 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.769+0000 7f61d5ffb700 1 -- 172.21.15.90:0/3182922742 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) v1 ==== 54+0+317 (secure 0 0 0) 0x7f61cc01b5f0 con 0x7f61d8105510 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 msgr2=0x7f61d819c860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:14.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 --2- 172.21.15.90:0/3182922742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f61cc004000 tx=0x7f61cc004750 comp rx=0 tx=0).stop 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 shutdown_connections 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 --2- 172.21.15.90:0/3182922742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d8105510 0x7f61d819c860 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 >> 172.21.15.90:0/3182922742 conn(0x7f61d8100a90 msgr2=0x7f61d818f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 shutdown_connections 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:13.770+0000 7f61def0d700 1 -- 172.21.15.90:0/3182922742 wait complete. 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:mon is available 2024-04-06T13:58:14.158 INFO:teuthology.orchestra.run.smithi090.stdout:Assimilating anything we can from ceph.conf... 2024-04-06T13:58:14.351 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-06T13:58:14.351 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: monmap e1: 1 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:58:14.351 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: fsmap 2024-04-06T13:58:14.351 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: osdmap e1: 0 total, 0 up, 0 in 2024-04-06T13:58:14.351 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: mgrmap e1: no daemons active 2024-04-06T13:58:14.352 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:14 smithi090 ceph-mon[42936]: from='client.? 172.21.15.90:0/3182922742' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-06T13:58:15.026 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [global] 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout fsid = 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.90:3300,v1:172.21.15.90:6789] 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [osd] 2024-04-06T13:58:15.027 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c1b817700 1 Processor -- start 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c1b817700 1 -- start start 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c1b817700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c1b817700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c14079e60 con 0x7f7c141070d0 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c195b3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c195b3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45970/0 (socket says 172.21.15.90:45970) 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c195b3700 1 -- 172.21.15.90:0/3107211328 learned_addr learned my addr 172.21.15.90:0/3107211328 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.584+0000 7f7c195b3700 1 -- 172.21.15.90:0/3107211328 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c14109a00 con 0x7f7c141070d0 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c195b3700 1 --2- 172.21.15.90:0/3107211328 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f7c100098a0 tx=0x7f7c10009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e1cf4075d3e87c39 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:15.028 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c0bfff700 1 -- 172.21.15.90:0/3107211328 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c10004160 con 0x7f7c141070d0 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c0bfff700 1 -- 172.21.15.90:0/3107211328 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7c100042c0 con 0x7f7c141070d0 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c0bfff700 1 -- 172.21.15.90:0/3107211328 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c100045a0 con 0x7f7c141070d0 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 -- 172.21.15.90:0/3107211328 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 msgr2=0x7f7c141094c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 --2- 172.21.15.90:0/3107211328 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f7c100098a0 tx=0x7f7c10009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 -- 172.21.15.90:0/3107211328 shutdown_connections 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 --2- 172.21.15.90:0/3107211328 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c141094c0 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 -- 172.21.15.90:0/3107211328 >> 172.21.15.90:0/3107211328 conn(0x7f7c14100aa0 msgr2=0x7f7c14102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 -- 172.21.15.90:0/3107211328 shutdown_connections 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.585+0000 7f7c1b817700 1 -- 172.21.15.90:0/3107211328 wait complete. 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c1b817700 1 Processor -- start 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c1b817700 1 -- start start 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c1b817700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:15.029 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c1b817700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c14105080 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c195b3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c195b3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45974/0 (socket says 172.21.15.90:45974) 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c195b3700 1 -- 172.21.15.90:0/2861006166 learned_addr learned my addr 172.21.15.90:0/2861006166 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.586+0000 7f7c195b3700 1 -- 172.21.15.90:0/2861006166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c10009550 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c195b3700 1 --2- 172.21.15.90:0/2861006166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f7c140792a0 tx=0x7f7c10004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c10011630 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f7c10011790 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c10011a30 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c14105280 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c14105720 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f7c10021070 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.587+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f7c1001b440 con 0x7f7c141070d0 2024-04-06T13:58:15.030 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.588+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7c14193900 con 0x7f7c141070d0 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.590+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f7c1001a460 con 0x7f7c141070d0 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.655+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f7c14075420 con 0x7f7c141070d0 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.659+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 7 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7c1001a600 con 0x7f7c141070d0 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.659+0000 7f7c0a7fc700 1 -- 172.21.15.90:0/2861006166 <== mon.0 v2:172.21.15.90:3300/0 8 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) v1 ==== 70+0+429 (secure 0 0 0) 0x7f7c1002c6e0 con 0x7f7c141070d0 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 msgr2=0x7f7c14106ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 --2- 172.21.15.90:0/2861006166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f7c140792a0 tx=0x7f7c10004750 comp rx=0 tx=0).stop 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 shutdown_connections 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 --2- 172.21.15.90:0/2861006166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c141070d0 0x7f7c14106ae0 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 >> 172.21.15.90:0/2861006166 conn(0x7f7c14100aa0 msgr2=0x7f7c14102230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 shutdown_connections 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:14.660+0000 7f7c1b817700 1 -- 172.21.15.90:0/2861006166 wait complete. 2024-04-06T13:58:15.031 INFO:teuthology.orchestra.run.smithi090.stdout:Generating new minimal ceph.conf... 2024-04-06T13:58:15.783 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:15 smithi090 ceph-mon[42936]: from='client.? 172.21.15.90:0/2861006166' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-06T13:58:15.783 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:15 smithi090 ceph-mon[42936]: from='client.? 172.21.15.90:0/2861006166' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-06T13:58:16.375 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e78d3a700 1 Processor -- start 2024-04-06T13:58:16.375 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e78d3a700 1 -- start start 2024-04-06T13:58:16.375 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e78d3a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:16.375 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e78d3a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e74107020 con 0x7f9e74106630 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e7259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e7259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45978/0 (socket says 172.21.15.90:45978) 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e7259c700 1 -- 172.21.15.90:0/856918938 learned_addr learned my addr 172.21.15.90:0/856918938 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.433+0000 7f9e7259c700 1 -- 172.21.15.90:0/856918938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e741077e0 con 0x7f9e74106630 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e7259c700 1 --2- 172.21.15.90:0/856918938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f9e640098a0 tx=0x7f9e64009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=406a55bd889e09f3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e7159a700 1 -- 172.21.15.90:0/856918938 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9e64004160 con 0x7f9e74106630 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e7159a700 1 -- 172.21.15.90:0/856918938 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9e640042c0 con 0x7f9e74106630 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e7159a700 1 -- 172.21.15.90:0/856918938 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9e64011a00 con 0x7f9e74106630 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 -- 172.21.15.90:0/856918938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 msgr2=0x7f9e74106a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 --2- 172.21.15.90:0/856918938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f9e640098a0 tx=0x7f9e64009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 -- 172.21.15.90:0/856918938 shutdown_connections 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 --2- 172.21.15.90:0/856918938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e74106a50 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:16.376 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 -- 172.21.15.90:0/856918938 >> 172.21.15.90:0/856918938 conn(0x7f9e74101c30 msgr2=0x7f9e74104050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 -- 172.21.15.90:0/856918938 shutdown_connections 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.434+0000 7f9e78d3a700 1 -- 172.21.15.90:0/856918938 wait complete. 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e78d3a700 1 Processor -- start 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e78d3a700 1 -- start start 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e78d3a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e78d3a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e7419a9a0 con 0x7f9e74106630 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e7259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e7259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45994/0 (socket says 172.21.15.90:45994) 2024-04-06T13:58:16.377 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.435+0000 7f9e7259c700 1 -- 172.21.15.90:0/1999000527 learned_addr learned my addr 172.21.15.90:0/1999000527 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e7259c700 1 -- 172.21.15.90:0/1999000527 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e64009550 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e7259c700 1 --2- 172.21.15.90:0/1999000527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9e64009130 tx=0x7f9e64011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9e64003ab0 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e7419aba0 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9e64003c10 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9e640247d0 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e7419b700 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f9e64024d50 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.436+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f9e640193c0 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.437+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e7419b040 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.439+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f9e7419b040 con 0x7f9e74106630 2024-04-06T13:58:16.378 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.506+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f9e74075420 con 0x7f9e74106630 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.506+0000 7f9e637fe700 1 -- 172.21.15.90:0/1999000527 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) v1 ==== 76+0+175 (secure 0 0 0) 0x7f9e6401abb0 con 0x7f9e74106630 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 msgr2=0x7f9e7419a460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 --2- 172.21.15.90:0/1999000527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f9e64009130 tx=0x7f9e64011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 shutdown_connections 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 --2- 172.21.15.90:0/1999000527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e74106630 0x7f9e7419a460 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 >> 172.21.15.90:0/1999000527 conn(0x7f9e74101c30 msgr2=0x7f9e74102830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 shutdown_connections 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:15.507+0000 7f9e78d3a700 1 -- 172.21.15.90:0/1999000527 wait complete. 2024-04-06T13:58:16.379 INFO:teuthology.orchestra.run.smithi090.stdout:Restarting the monitor... 2024-04-06T13:58:16.581 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:16 smithi090 ceph-mon[42936]: from='client.? 172.21.15.90:0/1999000527' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:58:16.582 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:16 smithi090 systemd[1]: Stopping Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:58:17.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a[42913]: 2024-04-06T13:58:16.580+0000 7fb2b1b2d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T13:58:17.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a[42913]: 2024-04-06T13:58:16.580+0000 7fb2b1b2d700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-06T13:58:17.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:16 smithi090 bash[43263]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a 2024-04-06T13:58:17.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 systemd[1]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service: Succeeded. 2024-04-06T13:58:17.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 systemd[1]: Stopped Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:58:17.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 systemd[1]: Starting Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:58:17.388 INFO:teuthology.orchestra.run.smithi090.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-06T13:58:17.389 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 bash[43383]: 5fa712d373516029fc0575e209e5979f57d8d81ff27024c37e209116ef4236d8 2024-04-06T13:58:17.389 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 systemd[1]: Started Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:58:17.389 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: set uid:gid to 167:167 (ceph:ceph) 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable), process ceph-mon, pid 7 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: pidfile_write: ignore empty --pid-file 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: load: jerasure load: lrc 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: RocksDB version: 7.9.2 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Git sha 0 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Compile date 2024-04-03 02:24:10 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: DB SUMMARY 2024-04-06T13:58:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: DB Session ID: 2A57QWN7I8KUZ2IU37KL 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: CURRENT file: CURRENT 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: IDENTITY file: IDENTITY 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 99983 ; 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.error_if_exists: 0 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.create_if_missing: 0 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.paranoid_checks: 1 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.env: 0x55e58ce947a0 2024-04-06T13:58:17.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.fs: PosixFileSystem 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.info_log: 0x55e58f59b4e0 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_file_opening_threads: 16 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.statistics: (nil) 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.use_fsync: 0 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_log_file_size: 0 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.keep_log_file_num: 1000 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.recycle_log_file_num: 0 2024-04-06T13:58:17.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_fallocate: 1 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_mmap_reads: 0 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_mmap_writes: 0 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.use_direct_reads: 0 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.create_missing_column_families: 0 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.db_log_dir: 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.wal_dir: 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-06T13:58:17.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.advise_random_on_open: 1 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.db_write_buffer_size: 0 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.write_buffer_manager: 0x55e58e832320 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-06T13:58:17.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.rate_limiter: (nil) 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.wal_recovery_mode: 2 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enable_thread_tracking: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enable_pipelined_write: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.unordered_write: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.row_cache: None 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.wal_filter: None 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-06T13:58:17.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_ingest_behind: 0 2024-04-06T13:58:17.796 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.two_write_queues: 0 2024-04-06T13:58:17.796 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.manual_wal_flush: 0 2024-04-06T13:58:17.796 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.wal_compression: 0 2024-04-06T13:58:17.796 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.atomic_flush: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.log_readahead_size: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.best_efforts_recovery: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.allow_data_in_errors: 0 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.db_host_id: __hostname__ 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_background_jobs: 2 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_background_compactions: -1 2024-04-06T13:58:17.797 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_subcompactions: 1 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_total_wal_size: 0 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_open_files: -1 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bytes_per_sync: 0 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-06T13:58:17.798 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_readahead_size: 0 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_background_flushes: -1 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Compression algorithms supported: 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kZSTD supported: 0 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kXpressCompression supported: 0 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kLZ4HCCompression supported: 1 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kZlibCompression supported: 1 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kSnappyCompression supported: 1 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kLZ4Compression supported: 1 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: kBZip2Compression supported: 0 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-06T13:58:17.799 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.merge_operator: 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_filter: None 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_filter_factory: None 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.sst_partitioner_factory: None 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e58f59b5e0) 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks: 1 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: pin_top_level_index_and_filter: 1 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_type: 0 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_index_type: 0 2024-04-06T13:58:17.800 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_shortening: 1 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: checksum: 4 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: no_block_cache: 0 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache: 0x55e58e8b31f0 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_name: BinnedLRUCache 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_options: 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: capacity : 536870912 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: num_shard_bits : 4 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: strict_capacity_limit : 0 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: high_pri_pool_ratio: 0.000 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_cache_compressed: (nil) 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: persistent_cache: (nil) 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size: 4096 2024-04-06T13:58:17.801 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_size_deviation: 10 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_restart_interval: 16 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: index_block_restart_interval: 1 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: metadata_block_size: 4096 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: partition_filters: 0 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: use_delta_encoding: 1 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: filter_policy: bloomfilter 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: whole_key_filtering: 1 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: verify_compression: 0 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: read_amp_bytes_per_bit: 0 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: format_version: 5 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_index_compression: 1 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: block_align: 0 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: max_auto_readahead_size: 262144 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: prepopulate_block_cache: 0 2024-04-06T13:58:17.802 INFO:journalctl@ceph.mon.a.smithi090.stdout: initial_auto_readahead_size: 8192 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout: num_file_reads_for_auto_readahead: 2 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.write_buffer_size: 33554432 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_write_buffer_number: 2 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression: NoCompression 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression: Disabled 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.prefix_extractor: nullptr 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.num_levels: 7 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-06T13:58:17.803 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-06T13:58:17.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-06T13:58:17.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-06T13:58:17.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-06T13:58:17.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-06T13:58:17.805 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.level: 32767 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.strategy: 0 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.enabled: false 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-06T13:58:17.806 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.target_file_size_base: 67108864 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-06T13:58:17.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.arena_block_size: 1048576 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.disable_auto_compactions: 0 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-06T13:58:17.808 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.table_properties_collectors: 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.inplace_update_support: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.bloom_locality: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.max_successive_merges: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.paranoid_file_checks: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.force_consistency_checks: 1 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.report_bg_io_stats: 0 2024-04-06T13:58:17.809 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.ttl: 2592000 2024-04-06T13:58:17.810 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enable_blob_files: false 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.min_blob_size: 0 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_file_size: 268435456 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-06T13:58:17.811 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.blob_file_starting_level: 0 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ea409d1a-bf32-4d61-b40a-21b60e34eaa1 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411897396235, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411897397225, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 95164, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 318, "table_properties": {"data_size": 93152, "index_size": 294, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 13914, "raw_average_key_size": 50, "raw_value_size": 85540, "raw_average_value_size": 308, "num_data_blocks": 11, "num_entries": 277, "num_filter_entries": 277, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712411897, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ea409d1a-bf32-4d61-b40a-21b60e34eaa1", "db_session_id": "2A57QWN7I8KUZ2IU37KL", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411897397296, "job": 1, "event": "recovery_finished"} 2024-04-06T13:58:17.812 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e58e950000 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: DB pointer 0x55e58e93e000 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-06T13:58:17.813 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** DB Stats ** 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: L0 2/0 94.77 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-06T13:58:17.814 INFO:journalctl@ceph.mon.a.smithi090.stdout: Sum 2/0 94.77 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 154.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** Compaction Stats [default] ** 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 154.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-06T13:58:17.815 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: Cumulative compaction: 0.00 GB write, 26.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: Interval compaction: 0.00 GB write, 26.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: Block cache BinnedLRUCache@0x55e58e8b31f0#7 capacity: 512.00 MB usage: 1.39 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.89 KB,0.000169873%) IndexBlock(2,0.50 KB,9.53674e-05%) Misc(1,0.00 KB,0%) 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: starting mon.a rank 0 at public addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] at bind addrs [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???) e1 preinit fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).mds e1 new map 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).mds e1 print_map 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: e1 2024-04-06T13:58:17.816 INFO:journalctl@ceph.mon.a.smithi090.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout: legacy client fscid: -1 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout: 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout: No filesystems configured 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: monmap e1: 1 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: fsmap 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: osdmap e1: 0 total, 0 up, 0 in 2024-04-06T13:58:17.817 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:17 smithi090 ceph-mon[43420]: mgrmap e1: no daemons active 2024-04-06T13:58:18.170 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.730+0000 7f275cdff700 1 Processor -- start 2024-04-06T13:58:18.175 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.730+0000 7f275cdff700 1 -- start start 2024-04-06T13:58:18.175 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275cdff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:18.175 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275cdff700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27581052d0 con 0x7f27581048e0 2024-04-06T13:58:18.175 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45996/0 (socket says 172.21.15.90:45996) 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275659c700 1 -- 172.21.15.90:0/2013451298 learned_addr learned my addr 172.21.15.90:0/2013451298 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275659c700 1 -- 172.21.15.90:0/2013451298 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2758107870 con 0x7f27581048e0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.731+0000 7f275659c700 1 --2- 172.21.15.90:0/2013451298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f274c0098a0 tx=0x7f274c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8cef2eb7220d8201 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275559a700 1 -- 172.21.15.90:0/2013451298 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f274c004160 con 0x7f27581048e0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275559a700 1 -- 172.21.15.90:0/2013451298 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f274c0042c0 con 0x7f27581048e0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275559a700 1 -- 172.21.15.90:0/2013451298 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f274c011a00 con 0x7f27581048e0 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 -- 172.21.15.90:0/2013451298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 msgr2=0x7f2758104d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 --2- 172.21.15.90:0/2013451298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f274c0098a0 tx=0x7f274c009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 -- 172.21.15.90:0/2013451298 shutdown_connections 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 --2- 172.21.15.90:0/2013451298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f2758104d00 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 -- 172.21.15.90:0/2013451298 >> 172.21.15.90:0/2013451298 conn(0x7f27581002b0 msgr2=0x7f27581026d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:18.176 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 -- 172.21.15.90:0/2013451298 shutdown_connections 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.732+0000 7f275cdff700 1 -- 172.21.15.90:0/2013451298 wait complete. 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275cdff700 1 Processor -- start 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275cdff700 1 -- start start 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275cdff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275cdff700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f275819a990 con 0x7f27581048e0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46010/0 (socket says 172.21.15.90:46010) 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.733+0000 7f275659c700 1 -- 172.21.15.90:0/3135456281 learned_addr learned my addr 172.21.15.90:0/3135456281 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f275659c700 1 -- 172.21.15.90:0/3135456281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f274c009550 con 0x7f27581048e0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f275659c700 1 --2- 172.21.15.90:0/3135456281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f274c009fb0 tx=0x7f274c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f274c003a40 con 0x7f27581048e0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f275819ab90 con 0x7f27581048e0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f274c003ba0 con 0x7f27581048e0 2024-04-06T13:58:18.177 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f274c01a750 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f275819b6f0 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f274c01acd0 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.734+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f274c0193c0 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.735+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f275819b030 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.737+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f275819b030 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.808+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) v1 -- 0x7f275810aa70 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.811+0000 7f27477fe700 1 -- 172.21.15.90:0/3135456281 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) v1 ==== 124+0+0 (secure 0 0 0) 0x7f275810aa70 con 0x7f27581048e0 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 msgr2=0x7f275819a450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 --2- 172.21.15.90:0/3135456281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f274c009fb0 tx=0x7f274c011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 shutdown_connections 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 --2- 172.21.15.90:0/3135456281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27581048e0 0x7f275819a450 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 >> 172.21.15.90:0/3135456281 conn(0x7f27581002b0 msgr2=0x7f2758100e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:18.178 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 shutdown_connections 2024-04-06T13:58:18.179 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:17.812+0000 7f275cdff700 1 -- 172.21.15.90:0/3135456281 wait complete. 2024-04-06T13:58:18.179 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-06T13:58:18.193 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-06T13:58:18.193 INFO:teuthology.orchestra.run.smithi090.stdout:Creating mgr... 2024-04-06T13:58:18.193 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying port 0.0.0.0:9283 ... 2024-04-06T13:58:18.193 INFO:teuthology.orchestra.run.smithi090.stdout:Verifying port 0.0.0.0:8765 ... 2024-04-06T13:58:18.408 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x 2024-04-06T13:58:18.409 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Failed to reset failed state of unit ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x.service: Unit ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x.service not loaded. 2024-04-06T13:58:18.503 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4.target.wants/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x.service → /etc/systemd/system/ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@.service. 2024-04-06T13:58:18.751 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:18 smithi090 systemd[1]: Starting Ceph mgr.x for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:58:18.907 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-06T13:58:18.907 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-06T13:58:18.915 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-06T13:58:18.915 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:58:18.915 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-06T13:58:18.915 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to enable service . firewalld.service is not available 2024-04-06T13:58:18.923 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-06T13:58:18.923 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr to start... 2024-04-06T13:58:18.931 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr... 2024-04-06T13:58:19.062 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:18 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3135456281' entity='client.admin' 2024-04-06T13:58:19.062 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:18 smithi090 bash[43658]: 1fc7301132dc3992098ec9cf86209c0b0c678f12622154821be27e770a921e56 2024-04-06T13:58:19.062 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:18 smithi090 systemd[1]: Started Ceph mgr.x for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:58:19.396 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:19.295+0000 7ff634a1b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T13:58:19.396 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:19.394+0000 7ff634a1b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T13:58:19.706 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:19.487+0000 7ff634a1b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T13:58:19.706 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:19.574+0000 7ff634a1b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T13:58:19.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:19.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:19.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 1, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-06T13:58:19.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-06T13:58:19.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-06T13:58:13.322384+0000", 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.313+0000 7f438ac55700 1 Processor -- start 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.313+0000 7f438ac55700 1 -- start start 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.313+0000 7f438ac55700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.313+0000 7f438ac55700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4384105f00 con 0x7f4384105510 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43889f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43889f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46038/0 (socket says 172.21.15.90:46038) 2024-04-06T13:58:19.760 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43889f1700 1 -- 172.21.15.90:0/1520304643 learned_addr learned my addr 172.21.15.90:0/1520304643 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43889f1700 1 -- 172.21.15.90:0/1520304643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4384106710 con 0x7f4384105510 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43889f1700 1 --2- 172.21.15.90:0/1520304643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f43780098a0 tx=0x7f4378009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=315b0a54da9b317f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43837fe700 1 -- 172.21.15.90:0/1520304643 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4378004160 con 0x7f4384105510 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43837fe700 1 -- 172.21.15.90:0/1520304643 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f43780042c0 con 0x7f4384105510 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.314+0000 7f43837fe700 1 -- 172.21.15.90:0/1520304643 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4378011a00 con 0x7f4384105510 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 -- 172.21.15.90:0/1520304643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 msgr2=0x7f4384105930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 --2- 172.21.15.90:0/1520304643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f43780098a0 tx=0x7f4378009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 -- 172.21.15.90:0/1520304643 shutdown_connections 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 --2- 172.21.15.90:0/1520304643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f4384105930 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 -- 172.21.15.90:0/1520304643 >> 172.21.15.90:0/1520304643 conn(0x7f4384100a90 msgr2=0x7f4384102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 -- 172.21.15.90:0/1520304643 shutdown_connections 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.315+0000 7f438ac55700 1 -- 172.21.15.90:0/1520304643 wait complete. 2024-04-06T13:58:19.761 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f438ac55700 1 Processor -- start 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f438ac55700 1 -- start start 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f438ac55700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f438ac55700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f438419a9a0 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f43889f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f43889f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46042/0 (socket says 172.21.15.90:46042) 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f43889f1700 1 -- 172.21.15.90:0/2260369878 learned_addr learned my addr 172.21.15.90:0/2260369878 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f43889f1700 1 -- 172.21.15.90:0/2260369878 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4378009550 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f43889f1700 1 --2- 172.21.15.90:0/2260369878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f4378009fb0 tx=0x7f4378011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.316+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4378003a40 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4378003ba0 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f438419aba0 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f437801a750 con 0x7f4384105510 2024-04-06T13:58:19.762 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f438419b700 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f437801acd0 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.317+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f43780193c0 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.318+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f438419b040 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.320+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f438419b040 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.392+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f43841047a0 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.392+0000 7f4381ffb700 1 -- 172.21.15.90:0/2260369878 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7f4378024b60 con 0x7f4384105510 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 msgr2=0x7f438419a460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 --2- 172.21.15.90:0/2260369878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f4378009fb0 tx=0x7f4378011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 shutdown_connections 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 --2- 172.21.15.90:0/2260369878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4384105510 0x7f438419a460 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 >> 172.21.15.90:0/2260369878 conn(0x7f4384100a90 msgr2=0x7f43841019f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 shutdown_connections 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:19.393+0000 7f438ac55700 1 -- 172.21.15.90:0/2260369878 wait complete. 2024-04-06T13:58:19.763 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (1/15)... 2024-04-06T13:58:20.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:19 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2260369878' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-06T13:58:20.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.214+0000 7ff634a1b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:58:20.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.460+0000 7ff634a1b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T13:58:20.809 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.544+0000 7ff634a1b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T13:58:20.809 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.652+0000 7ff634a1b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T13:58:21.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.807+0000 7ff634a1b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T13:58:21.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:20.933+0000 7ff634a1b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T13:58:21.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:21.017+0000 7ff634a1b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T13:58:21.766 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:21.382+0000 7ff634a1b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T13:58:22.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:21.765+0000 7ff634a1b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T13:58:22.039 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:21.974+0000 7ff634a1b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T13:58:22.385 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:22 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3767432658' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-06T13:58:22.385 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:22.070+0000 7ff634a1b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-06T13:58:22.597 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 4, 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-06T13:58:22.598 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-06T13:58:22.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-06T13:58:22.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-06T13:58:13.322384+0000", 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:22.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca18097700 1 Processor -- start 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca18097700 1 -- start start 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca18097700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca18097700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca10107020 con 0x7fca10106630 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca15e33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca15e33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46046/0 (socket says 172.21.15.90:46046) 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.145+0000 7fca15e33700 1 -- 172.21.15.90:0/71121256 learned_addr learned my addr 172.21.15.90:0/71121256 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.146+0000 7fca15e33700 1 -- 172.21.15.90:0/71121256 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca101077e0 con 0x7fca10106630 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.146+0000 7fca15e33700 1 --2- 172.21.15.90:0/71121256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fca0c0098a0 tx=0x7fca0c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1da89e9ddbe913bf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.146+0000 7fca14e31700 1 -- 172.21.15.90:0/71121256 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca0c004160 con 0x7fca10106630 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.146+0000 7fca14e31700 1 -- 172.21.15.90:0/71121256 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fca0c0042c0 con 0x7fca10106630 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.146+0000 7fca14e31700 1 -- 172.21.15.90:0/71121256 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca0c011a00 con 0x7fca10106630 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 -- 172.21.15.90:0/71121256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 msgr2=0x7fca10106a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:22.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 --2- 172.21.15.90:0/71121256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fca0c0098a0 tx=0x7fca0c009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 -- 172.21.15.90:0/71121256 shutdown_connections 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 --2- 172.21.15.90:0/71121256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca10106a50 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 -- 172.21.15.90:0/71121256 >> 172.21.15.90:0/71121256 conn(0x7fca10101c30 msgr2=0x7fca10104050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 -- 172.21.15.90:0/71121256 shutdown_connections 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 -- 172.21.15.90:0/71121256 wait complete. 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.147+0000 7fca18097700 1 Processor -- start 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca18097700 1 -- start start 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca18097700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca18097700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca1019a9e0 con 0x7fca10106630 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca15e33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca15e33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46060/0 (socket says 172.21.15.90:46060) 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca15e33700 1 -- 172.21.15.90:0/3767432658 learned_addr learned my addr 172.21.15.90:0/3767432658 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca15e33700 1 -- 172.21.15.90:0/3767432658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca0c009550 con 0x7fca10106630 2024-04-06T13:58:22.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca15e33700 1 --2- 172.21.15.90:0/3767432658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fca0c009130 tx=0x7fca0c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca0c003ab0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fca0c003c10 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca1019abe0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.148+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca0c0247d0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.149+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca1019b740 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.149+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fca0c024d00 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.149+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fca0c0193c0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.149+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca1019b080 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.151+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7fca1019b080 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.222+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7fca101058c0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.222+0000 7fca06ffd700 1 -- 172.21.15.90:0/3767432658 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7fca0c01abb0 con 0x7fca10106630 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 msgr2=0x7fca1019a4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 --2- 172.21.15.90:0/3767432658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fca0c009130 tx=0x7fca0c011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 shutdown_connections 2024-04-06T13:58:22.604 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 --2- 172.21.15.90:0/3767432658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca10106630 0x7fca1019a4a0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:22.605 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 >> 172.21.15.90:0/3767432658 conn(0x7fca10101c30 msgr2=0x7fca101027e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:22.605 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 shutdown_connections 2024-04-06T13:58:22.605 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:22.223+0000 7fca18097700 1 -- 172.21.15.90:0/3767432658 wait complete. 2024-04-06T13:58:22.605 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (2/15)... 2024-04-06T13:58:23.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:22.850+0000 7ff634a1b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T13:58:23.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:22.946+0000 7ff634a1b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T13:58:23.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:23.092+0000 7ff634a1b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T13:58:24.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:23.707+0000 7ff634a1b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T13:58:24.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:23.888+0000 7ff634a1b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T13:58:24.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:24.080+0000 7ff634a1b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T13:58:25.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:25 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2549540499' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-06T13:58:25.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:25.019+0000 7ff634a1b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-06T13:58:25.389 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-06T13:58:25.390 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-06T13:58:25.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-06T13:58:25.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-06T13:58:13.322384+0000", 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85ba777700 1 Processor -- start 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85ba777700 1 -- start start 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85ba777700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85ba777700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85b4107000 con 0x7f85b4106610 2024-04-06T13:58:25.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55306/0 (socket says 172.21.15.90:55306) 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85b3fff700 1 -- 172.21.15.90:0/165373811 learned_addr learned my addr 172.21.15.90:0/165373811 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.960+0000 7f85b3fff700 1 -- 172.21.15.90:0/165373811 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f85b41077c0 con 0x7f85b4106610 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85b3fff700 1 --2- 172.21.15.90:0/165373811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f85a40098a0 tx=0x7f85a4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6755444980990bfc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85b2ffd700 1 -- 172.21.15.90:0/165373811 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f85a4004160 con 0x7f85b4106610 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85b2ffd700 1 -- 172.21.15.90:0/165373811 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f85a40042c0 con 0x7f85b4106610 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85b2ffd700 1 -- 172.21.15.90:0/165373811 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f85a4011a00 con 0x7f85b4106610 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85ba777700 1 -- 172.21.15.90:0/165373811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 msgr2=0x7f85b4106a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85ba777700 1 --2- 172.21.15.90:0/165373811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f85a40098a0 tx=0x7f85a4009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85ba777700 1 -- 172.21.15.90:0/165373811 shutdown_connections 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85ba777700 1 --2- 172.21.15.90:0/165373811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b4106a30 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.961+0000 7f85ba777700 1 -- 172.21.15.90:0/165373811 >> 172.21.15.90:0/165373811 conn(0x7f85b4101bd0 msgr2=0x7f85b4103ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:25.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 -- 172.21.15.90:0/165373811 shutdown_connections 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 -- 172.21.15.90:0/165373811 wait complete. 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 Processor -- start 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 -- start start 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.962+0000 7f85ba777700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85b407a2b0 con 0x7f85b4106610 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55312/0 (socket says 172.21.15.90:55312) 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b3fff700 1 -- 172.21.15.90:0/2549540499 learned_addr learned my addr 172.21.15.90:0/2549540499 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b3fff700 1 -- 172.21.15.90:0/2549540499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f85a4009550 con 0x7f85b4106610 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b3fff700 1 --2- 172.21.15.90:0/2549540499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f85a4009fb0 tx=0x7f85a4011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f85a4003a40 con 0x7f85b4106610 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f85b407a4b0 con 0x7f85b4106610 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f85a4003ba0 con 0x7f85b4106610 2024-04-06T13:58:25.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f85a401a750 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f85b407a950 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.963+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f85a401acd0 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.964+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f85a40193c0 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.964+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f85b41058a0 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:24.966+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72706 (secure 0 0 0) 0x7f85a402de40 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.037+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f85b4075470 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.038+0000 7f85b17fa700 1 -- 172.21.15.90:0/2549540499 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7f85a4024b60 con 0x7f85b4106610 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 msgr2=0x7f85b407bd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 --2- 172.21.15.90:0/2549540499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f85a4009fb0 tx=0x7f85a4011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 shutdown_connections 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 --2- 172.21.15.90:0/2549540499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f85b4106610 0x7f85b407bd10 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 >> 172.21.15.90:0/2549540499 conn(0x7f85b4101bd0 msgr2=0x7f85b41027d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 shutdown_connections 2024-04-06T13:58:25.404 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:25.039+0000 7f85ba777700 1 -- 172.21.15.90:0/2549540499 wait complete. 2024-04-06T13:58:25.405 INFO:teuthology.orchestra.run.smithi090.stdout:mgr not available, waiting (3/15)... 2024-04-06T13:58:25.595 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:25.315+0000 7ff634a1b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:58:25.595 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:25.406+0000 7ff634a1b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T13:58:26.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:25.593+0000 7ff634a1b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T13:58:26.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: Activating manager daemon x 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: mgrmap e2: x(active, starting, since 0.0032713s) 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: Manager daemon x is now available 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' 2024-04-06T13:58:26.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:26 smithi090 ceph-mon[43420]: from='mgr.14100 172.21.15.90:0/1908146504' entity='mgr.x' 2024-04-06T13:58:27.984 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:27 smithi090 ceph-mon[43420]: mgrmap e3: x(active, since 1.00667s) 2024-04-06T13:58:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:29 smithi090 ceph-mon[43420]: mgrmap e4: x(active, since 3s) 2024-04-06T13:58:30.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:29 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2449857800' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-06T13:58:30.750 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:30.750 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "health": { 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 0 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "a" 2024-04-06T13:58:30.751 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-06T13:58:30.752 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-06T13:58:30.753 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "restful" 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ], 2024-04-06T13:58:30.754 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "modified": "2024-04-06T13:58:13.322384+0000", 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout }, 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.337+0000 7fac20d9a700 1 Processor -- start 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.337+0000 7fac20d9a700 1 -- start start 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.337+0000 7fac20d9a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.337+0000 7fac20d9a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac1c079ec0 con 0x7fac1c107080 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.337+0000 7fac1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:30.755 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55378/0 (socket says 172.21.15.90:55378) 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1a59c700 1 -- 172.21.15.90:0/4126768231 learned_addr learned my addr 172.21.15.90:0/4126768231 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1a59c700 1 -- 172.21.15.90:0/4126768231 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac1c1099b0 con 0x7fac1c107080 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1a59c700 1 --2- 172.21.15.90:0/4126768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fac100098a0 tx=0x7fac10009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8965771f234ba470 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1959a700 1 -- 172.21.15.90:0/4126768231 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fac10004160 con 0x7fac1c107080 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1959a700 1 -- 172.21.15.90:0/4126768231 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fac100042c0 con 0x7fac1c107080 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.338+0000 7fac1959a700 1 -- 172.21.15.90:0/4126768231 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fac10011a00 con 0x7fac1c107080 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 -- 172.21.15.90:0/4126768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 msgr2=0x7fac1c109470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 --2- 172.21.15.90:0/4126768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fac100098a0 tx=0x7fac10009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 -- 172.21.15.90:0/4126768231 shutdown_connections 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 --2- 172.21.15.90:0/4126768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c109470 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 -- 172.21.15.90:0/4126768231 >> 172.21.15.90:0/4126768231 conn(0x7fac1c100a90 msgr2=0x7fac1c102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 -- 172.21.15.90:0/4126768231 shutdown_connections 2024-04-06T13:58:30.756 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.339+0000 7fac20d9a700 1 -- 172.21.15.90:0/4126768231 wait complete. 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac20d9a700 1 Processor -- start 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac20d9a700 1 -- start start 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac20d9a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac20d9a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac1c105030 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55390/0 (socket says 172.21.15.90:55390) 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac1a59c700 1 -- 172.21.15.90:0/2449857800 learned_addr learned my addr 172.21.15.90:0/2449857800 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac1a59c700 1 -- 172.21.15.90:0/2449857800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac10009550 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac1a59c700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fac10009fb0 tx=0x7fac10011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fac10003a40 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fac1c105230 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fac10003ba0 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.340+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fac1001a750 con 0x7fac1c107080 2024-04-06T13:58:30.757 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.341+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fac1c1056d0 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.341+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 3) v1 ==== 45424+0+0 (secure 0 0 0) 0x7fac1001a9d0 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.342+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fac1c075420 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.342+0000 7fac0b7fe700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 0x7fac0403aa00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.342+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fac1004c130 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.342+0000 7fac19d9b700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 0x7fac0403aa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.342+0000 7fac19d9b700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 0x7fac0403aa00 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fac0c006fd0 tx=0x7fac0c006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.347+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fac100189a0 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.615+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 4) v1 ==== 45530+0+0 (secure 0 0 0) 0x7fac1002d960 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.623+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7fac1c10a7b0 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.623+0000 7fac0b7fe700 1 -- 172.21.15.90:0/2449857800 <== mon.0 v2:172.21.15.90:3300/0 8 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7fac10018b80 con 0x7fac1c107080 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.625+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 msgr2=0x7fac0403aa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.625+0000 7fac20d9a700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 0x7fac0403aa00 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fac0c006fd0 tx=0x7fac0c006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:30.758 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.625+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 msgr2=0x7fac1c106a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.625+0000 7fac20d9a700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fac10009fb0 tx=0x7fac10011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 shutdown_connections 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fac04038540 0x7fac0403aa00 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 --2- 172.21.15.90:0/2449857800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fac1c107080 0x7fac1c106a90 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 >> 172.21.15.90:0/2449857800 conn(0x7fac1c100a90 msgr2=0x7fac1c101640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 shutdown_connections 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:28.626+0000 7fac20d9a700 1 -- 172.21.15.90:0/2449857800 wait complete. 2024-04-06T13:58:30.759 INFO:teuthology.orchestra.run.smithi090.stdout:mgr is available 2024-04-06T13:58:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:33 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2754692855' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-06T13:58:34.351 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [global] 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout fsid = 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 2024-04-06T13:58:34.352 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout [osd] 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.348+0000 7f5677b48700 1 Processor -- start 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.348+0000 7f5677b48700 1 -- start start 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.348+0000 7f5677b48700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.348+0000 7f5677b48700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5670079ec0 con 0x7f5670107070 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56758e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56758e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55396/0 (socket says 172.21.15.90:55396) 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56758e4700 1 -- 172.21.15.90:0/748864332 learned_addr learned my addr 172.21.15.90:0/748864332 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:34.353 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56758e4700 1 -- 172.21.15.90:0/748864332 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56701099a0 con 0x7f5670107070 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56758e4700 1 --2- 172.21.15.90:0/748864332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f56640098a0 tx=0x7f5664009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ddb6aa68900b9f7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56748e2700 1 -- 172.21.15.90:0/748864332 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5664004160 con 0x7f5670107070 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56748e2700 1 -- 172.21.15.90:0/748864332 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f56640042c0 con 0x7f5670107070 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.349+0000 7f56748e2700 1 -- 172.21.15.90:0/748864332 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5664011a00 con 0x7f5670107070 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 -- 172.21.15.90:0/748864332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 msgr2=0x7f5670109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 --2- 172.21.15.90:0/748864332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f56640098a0 tx=0x7f5664009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 -- 172.21.15.90:0/748864332 shutdown_connections 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 --2- 172.21.15.90:0/748864332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f5670109460 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 -- 172.21.15.90:0/748864332 >> 172.21.15.90:0/748864332 conn(0x7f5670100aa0 msgr2=0x7f5670102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 -- 172.21.15.90:0/748864332 shutdown_connections 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.350+0000 7f5677b48700 1 -- 172.21.15.90:0/748864332 wait complete. 2024-04-06T13:58:34.354 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f5677b48700 1 Processor -- start 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f5677b48700 1 -- start start 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f5677b48700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f5677b48700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f567019a9c0 con 0x7f5670107070 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f56758e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f56758e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55398/0 (socket says 172.21.15.90:55398) 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f56758e4700 1 -- 172.21.15.90:0/2754692855 learned_addr learned my addr 172.21.15.90:0/2754692855 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f56758e4700 1 -- 172.21.15.90:0/2754692855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5664009550 con 0x7f5670107070 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f56758e4700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f5664011a20 tx=0x7f5664011b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.351+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5664003af0 con 0x7f5670107070 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5664003c50 con 0x7f5670107070 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f567019abc0 con 0x7f5670107070 2024-04-06T13:58:34.355 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5664019760 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f567019b720 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 4) v1 ==== 45530+0+0 (secure 0 0 0) 0x7f5664019950 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.352+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5670104790 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.353+0000 7f5662ffd700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 0x7f565c03abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.353+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f566404ca00 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.353+0000 7f56750e3700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 0x7f565c03abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.353+0000 7f56750e3700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 0x7f565c03abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f566c006fd0 tx=0x7f566c006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.357+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5664018d90 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.549+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f5670075420 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.550+0000 7f5662ffd700 1 -- 172.21.15.90:0/2754692855 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) v1 ==== 70+0+373 (secure 0 0 0) 0x7f5664018350 con 0x7f5670107070 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 msgr2=0x7f565c03abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:34.356 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 0x7f565c03abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f566c006fd0 tx=0x7f566c006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 msgr2=0x7f567019a480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f5664011a20 tx=0x7f5664011b00 comp rx=0 tx=0).stop 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 shutdown_connections 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7f565c038700 0x7f565c03abc0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 --2- 172.21.15.90:0/2754692855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5670107070 0x7f567019a480 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 >> 172.21.15.90:0/2754692855 conn(0x7f5670100aa0 msgr2=0x7f56701016d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 shutdown_connections 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:32.552+0000 7f5677b48700 1 -- 172.21.15.90:0/2754692855 wait complete. 2024-04-06T13:58:34.357 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling cephadm module... 2024-04-06T13:58:36.464 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/62505363' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-06T13:58:36.464 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setuser ceph since I am not root 2024-04-06T13:58:36.464 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setgroup ceph since I am not root 2024-04-06T13:58:36.729 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:36.462+0000 7f897abca200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T13:58:36.729 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:36.557+0000 7f897abca200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T13:58:36.729 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:36.643+0000 7f897abca200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T13:58:36.730 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:36.728+0000 7f897abca200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T13:58:37.165 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f91eb700 1 Processor -- start 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f91eb700 1 -- start start 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f91eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f91eb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0f4079e00 con 0x7fb0f4107150 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47360/0 (socket says 172.21.15.90:47360) 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f2d9d700 1 -- 172.21.15.90:0/530284426 learned_addr learned my addr 172.21.15.90:0/530284426 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.729+0000 7fb0f2d9d700 1 -- 172.21.15.90:0/530284426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0f4079f40 con 0x7fb0f4107150 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f2d9d700 1 --2- 172.21.15.90:0/530284426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fb0e40098a0 tx=0x7fb0e4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4867407dc3da2da4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f1d9b700 1 -- 172.21.15.90:0/530284426 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0e4004160 con 0x7fb0f4107150 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f1d9b700 1 -- 172.21.15.90:0/530284426 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb0e40042c0 con 0x7fb0f4107150 2024-04-06T13:58:37.166 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f1d9b700 1 -- 172.21.15.90:0/530284426 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0e4011a00 con 0x7fb0f4107150 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 -- 172.21.15.90:0/530284426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 msgr2=0x7fb0f4109540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/530284426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7fb0e40098a0 tx=0x7fb0e4009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 -- 172.21.15.90:0/530284426 shutdown_connections 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/530284426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f4109540 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 -- 172.21.15.90:0/530284426 >> 172.21.15.90:0/530284426 conn(0x7fb0f4100b00 msgr2=0x7fb0f4102f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 -- 172.21.15.90:0/530284426 shutdown_connections 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.730+0000 7fb0f91eb700 1 -- 172.21.15.90:0/530284426 wait complete. 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.731+0000 7fb0f91eb700 1 Processor -- start 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.731+0000 7fb0f91eb700 1 -- start start 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.731+0000 7fb0f91eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.731+0000 7fb0f91eb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0f419a930 con 0x7fb0f4107150 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47368/0 (socket says 172.21.15.90:47368) 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f2d9d700 1 -- 172.21.15.90:0/62505363 learned_addr learned my addr 172.21.15.90:0/62505363 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f2d9d700 1 -- 172.21.15.90:0/62505363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0e4009550 con 0x7fb0f4107150 2024-04-06T13:58:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f2d9d700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fb0e4009130 tx=0x7fb0e4011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0e4003b70 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb0e4003cd0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0e40197d0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0f419ab30 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.732+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0f419b690 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.733+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0f41047b0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.734+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 4) v1 ==== 45530+0+0 (secure 0 0 0) 0x7fb0e401abb0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.734+0000 7fb0dbfff700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 0x7fb0dc03abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.734+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fb0e404cdc0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.734+0000 7fb0f259c700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 0x7fb0dc03abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.734+0000 7fb0f259c700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 0x7fb0dc03abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb0e8006fd0 tx=0x7fb0e8006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.738+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb0e4023370 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:35.994+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) v1 -- 0x7fb0f410a7b0 con 0x7fb0f4107150 2024-04-06T13:58:37.168 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.047+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 5) v1 ==== 45541+0+0 (secure 0 0 0) 0x7fb0e404bc00 con 0x7fb0f4107150 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.048+0000 7fb0dbfff700 1 -- 172.21.15.90:0/62505363 <== mon.0 v2:172.21.15.90:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) v1 ==== 86+0+0 (secure 0 0 0) 0x7fb0e404bf80 con 0x7fb0f4107150 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 msgr2=0x7fb0dc03abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 0x7fb0dc03abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb0e8006fd0 tx=0x7fb0e8006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 msgr2=0x7fb0f419a3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7fb0e4009130 tx=0x7fb0e4011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 shutdown_connections 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fb0dc038700 0x7fb0dc03abc0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 --2- 172.21.15.90:0/62505363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0f4107150 0x7fb0f419a3f0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 >> 172.21.15.90:0/62505363 conn(0x7fb0f4100b00 msgr2=0x7fb0f4101610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 shutdown_connections 2024-04-06T13:58:37.169 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:36.050+0000 7fb0f91eb700 1 -- 172.21.15.90:0/62505363 wait complete. 2024-04-06T13:58:37.275 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/62505363' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-06T13:58:37.275 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-mon[43420]: mgrmap e5: x(active, since 10s) 2024-04-06T13:58:37.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:37.373+0000 7f897abca200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:58:38.003 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:37.640+0000 7f897abca200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T13:58:38.003 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:37.727+0000 7f897abca200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T13:58:38.003 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:37.841+0000 7f897abca200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe633008700 1 Processor -- start 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe633008700 1 -- start start 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe633008700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe633008700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe62c105ef0 con 0x7fe62c105500 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe630da4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:38.184 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe630da4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47388/0 (socket says 172.21.15.90:47388) 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.551+0000 7fe630da4700 1 -- 172.21.15.90:0/525137343 learned_addr learned my addr 172.21.15.90:0/525137343 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe630da4700 1 -- 172.21.15.90:0/525137343 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe62c106700 con 0x7fe62c105500 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe630da4700 1 --2- 172.21.15.90:0/525137343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fe6200098a0 tx=0x7fe620009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c3d066e83f01eb6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe62b7fe700 1 -- 172.21.15.90:0/525137343 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe620004160 con 0x7fe62c105500 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe62b7fe700 1 -- 172.21.15.90:0/525137343 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe6200042c0 con 0x7fe62c105500 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe62b7fe700 1 -- 172.21.15.90:0/525137343 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe620011a00 con 0x7fe62c105500 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe633008700 1 -- 172.21.15.90:0/525137343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 msgr2=0x7fe62c105920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.552+0000 7fe633008700 1 --2- 172.21.15.90:0/525137343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fe6200098a0 tx=0x7fe620009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 -- 172.21.15.90:0/525137343 shutdown_connections 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 --2- 172.21.15.90:0/525137343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c105920 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 -- 172.21.15.90:0/525137343 >> 172.21.15.90:0/525137343 conn(0x7fe62c100aa0 msgr2=0x7fe62c102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 -- 172.21.15.90:0/525137343 shutdown_connections 2024-04-06T13:58:38.185 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 -- 172.21.15.90:0/525137343 wait complete. 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 Processor -- start 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.553+0000 7fe633008700 1 -- start start 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe633008700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe633008700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe62c199940 con 0x7fe62c105500 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe630da4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe630da4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47390/0 (socket says 172.21.15.90:47390) 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe630da4700 1 -- 172.21.15.90:0/1022485838 learned_addr learned my addr 172.21.15.90:0/1022485838 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe630da4700 1 -- 172.21.15.90:0/1022485838 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe620009550 con 0x7fe62c105500 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe630da4700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fe620009130 tx=0x7fe620011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe6200038b0 con 0x7fe62c105500 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe620003a10 con 0x7fe62c105500 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe62c199b40 con 0x7fe62c105500 2024-04-06T13:58:38.186 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.554+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe620019550 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.555+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe62c19a6a0 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.555+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 5) v1 ==== 45541+0+0 (secure 0 0 0) 0x7fe620019810 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.555+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe62c199fe0 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.556+0000 7fe629ffb700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 0x7fe61c03ab50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.556+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fe62004c450 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.556+0000 7fe62bfff700 1 -- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 msgr2=0x7fe61c03ab50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.556+0000 7fe62bfff700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 0x7fe61c03ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.564+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe62c199fe0 con 0x7fe62c105500 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.757+0000 7fe62bfff700 1 -- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 msgr2=0x7fe61c03ab50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.757+0000 7fe62bfff700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 0x7fe61c03ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T13:58:38.187 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.818+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7fe62c199fe0 con 0x7fe62c105500 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.819+0000 7fe629ffb700 1 -- 172.21.15.90:0/1022485838 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) v1 ==== 56+0+88 (secure 0 0 0) 0x7fe62c199fe0 con 0x7fe62c105500 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 msgr2=0x7fe61c03ab50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 0x7fe61c03ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 msgr2=0x7fe62c199400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fe620009130 tx=0x7fe620011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 shutdown_connections 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fe61c038690 0x7fe61c03ab50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 --2- 172.21.15.90:0/1022485838 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe62c105500 0x7fe62c199400 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 >> 172.21.15.90:0/1022485838 conn(0x7fe62c100aa0 msgr2=0x7fe62c1017e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 shutdown_connections 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:37.821+0000 7fe633008700 1 -- 172.21.15.90:0/1022485838 wait complete. 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for the mgr to restart... 2024-04-06T13:58:38.188 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr epoch 5... 2024-04-06T13:58:38.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:38.008+0000 7f897abca200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T13:58:38.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:38.138+0000 7f897abca200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T13:58:38.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:38.229+0000 7f897abca200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T13:58:38.615 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1022485838' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-06T13:58:38.997 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:38.614+0000 7f897abca200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T13:58:39.281 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:38.996+0000 7f897abca200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T13:58:39.281 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:39.188+0000 7f897abca200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T13:58:39.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:39.280+0000 7f897abca200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T13:58:40.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:40.033+0000 7f897abca200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T13:58:40.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:40.129+0000 7f897abca200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T13:58:40.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:40.276+0000 7f897abca200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T13:58:41.264 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:40.890+0000 7f897abca200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T13:58:41.265 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:41.071+0000 7f897abca200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T13:58:41.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:41.263+0000 7f897abca200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T13:58:42.467 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:42.181+0000 7f897abca200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T13:58:42.744 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:42.466+0000 7f897abca200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:58:42.745 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:42.554+0000 7f897abca200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T13:58:42.745 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:42.743+0000 7f897abca200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T13:58:43.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: Active manager daemon x restarted 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: Activating manager daemon x 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: osdmap e2: 0 total, 0 up, 0 in 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: mgrmap e6: x(active, starting, since 0.00330434s) 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: Manager daemon x is now available 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: Found migration_current of "None". Setting to last migration. 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:43.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:58:43.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:58:43.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T13:58:43.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:43 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T13:58:44.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:44.155 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.533+0000 7fd205f31700 1 Processor -- start 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.533+0000 7fd205f31700 1 -- start start 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd205f31700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd205f31700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd200079e60 con 0x7fd2001070d0 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1ff7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1ff7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47398/0 (socket says 172.21.15.90:47398) 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1ff7fe700 1 -- 172.21.15.90:0/3643147318 learned_addr learned my addr 172.21.15.90:0/3643147318 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1ff7fe700 1 -- 172.21.15.90:0/3643147318 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd200109a00 con 0x7fd2001070d0 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1ff7fe700 1 --2- 172.21.15.90:0/3643147318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd1f00098a0 tx=0x7fd1f0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7dfc3f13ff334141 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1fe7fc700 1 -- 172.21.15.90:0/3643147318 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd1f0004160 con 0x7fd2001070d0 2024-04-06T13:58:44.156 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1fe7fc700 1 -- 172.21.15.90:0/3643147318 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd1f00042c0 con 0x7fd2001070d0 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.534+0000 7fd1fe7fc700 1 -- 172.21.15.90:0/3643147318 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd1f0011a00 con 0x7fd2001070d0 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 -- 172.21.15.90:0/3643147318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 msgr2=0x7fd2001094c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 --2- 172.21.15.90:0/3643147318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd1f00098a0 tx=0x7fd1f0009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 -- 172.21.15.90:0/3643147318 shutdown_connections 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 --2- 172.21.15.90:0/3643147318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd2001094c0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 -- 172.21.15.90:0/3643147318 >> 172.21.15.90:0/3643147318 conn(0x7fd200100aa0 msgr2=0x7fd200102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 -- 172.21.15.90:0/3643147318 shutdown_connections 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.535+0000 7fd205f31700 1 -- 172.21.15.90:0/3643147318 wait complete. 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd205f31700 1 Processor -- start 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd205f31700 1 -- start start 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd205f31700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd205f31700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd20019aa30 con 0x7fd2001070d0 2024-04-06T13:58:44.157 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd1ff7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd1ff7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47414/0 (socket says 172.21.15.90:47414) 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd1ff7fe700 1 -- 172.21.15.90:0/1524074799 learned_addr learned my addr 172.21.15.90:0/1524074799 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd1ff7fe700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1f0009550 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.536+0000 7fd1ff7fe700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd1f0003eb0 tx=0x7fd1f0011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.537+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd1f00038b0 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.537+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd1f0003a10 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.537+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd20019ac30 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.537+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd1f0019420 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.537+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd20019b790 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 5) v1 ==== 45541+0+0 (secure 0 0 0) 0x7fd1f00196d0 con 0x7fd2001070d0 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1fcff9700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:44.158 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fd1e803b2d0 con 0x7fd1e8038700 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fd1f004d2b0 con 0x7fd2001070d0 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1feffd700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.538+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.739+0000 7fd1feffd700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:38.739+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:39.140+0000 7fd1feffd700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:39.140+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:39.941+0000 7fd1feffd700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:39.941+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:41.542+0000 7fd1feffd700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1491440652 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:41.542+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:42.759+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mgrmap(e 6) v1 ==== 45308+0+0 (secure 0 0 0) 0x7fd1f0019bb0 con 0x7fd2001070d0 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:42.759+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 msgr2=0x7fd1e803abc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:58:44.159 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:42.759+0000 7fd1fcff9700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.760+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 7) v1 ==== 45435+0+0 (secure 0 0 0) 0x7fd1f004de80 con 0x7fd2001070d0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.760+0000 7fd1fcff9700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 0x7fd1e803e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.760+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fd1e803b2d0 con 0x7fd1e803bfe0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.760+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 0x7fd1e803e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.761+0000 7fd1feffd700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 0x7fd1e803e3d0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7fd1f4003a10 tx=0x7fd1f40092b0 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.761+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6910 (secure 0 0 0) 0x7fd1e803b2d0 con 0x7fd1e803bfe0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7fd200104790 con 0x7fd1e803bfe0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd1fcff9700 1 -- 172.21.15.90:0/1524074799 <== mgr.14120 v2:172.21.15.90:6800/2246618981 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+51 (secure 0 0 0) 0x7fd200104790 con 0x7fd1e803bfe0 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 msgr2=0x7fd1e803e3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 0x7fd1e803e3d0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7fd1f4003a10 tx=0x7fd1f40092b0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 msgr2=0x7fd20019a4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd1f0003eb0 tx=0x7fd1f0011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.160 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 shutdown_connections 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd1e803bfe0 0x7fd1e803e3d0 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2001070d0 0x7fd20019a4f0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 --2- 172.21.15.90:0/1524074799 >> [v2:172.21.15.90:6800/1491440652,v1:172.21.15.90:6801/1491440652] conn(0x7fd1e8038700 0x7fd1e803abc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 >> 172.21.15.90:0/1524074799 conn(0x7fd200100aa0 msgr2=0x7fd200101760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.772+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 shutdown_connections 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:43.773+0000 7fd205f31700 1 -- 172.21.15.90:0/1524074799 wait complete. 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:mgr epoch 5 is available 2024-04-06T13:58:44.161 INFO:teuthology.orchestra.run.smithi090.stdout:Setting orchestrator backend to cephadm... 2024-04-06T13:58:44.988 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:44 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:44.988 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:44 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:44.988 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:44 smithi090 ceph-mon[43420]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-06T13:58:44.988 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:44 smithi090 ceph-mon[43420]: mgrmap e7: x(active, since 1.00551s) 2024-04-06T13:58:44.988 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:44 smithi090 ceph-mon[43420]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.476+0000 7f539f79d700 1 Processor -- start 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539f79d700 1 -- start start 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539f79d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539f79d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5398105ee0 con 0x7f53981054f0 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539d539700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539d539700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49096/0 (socket says 172.21.15.90:49096) 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.477+0000 7f539d539700 1 -- 172.21.15.90:0/4064800183 learned_addr learned my addr 172.21.15.90:0/4064800183 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.478+0000 7f539d539700 1 -- 172.21.15.90:0/4064800183 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53981066f0 con 0x7f53981054f0 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.478+0000 7f539d539700 1 --2- 172.21.15.90:0/4064800183 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f53940098a0 tx=0x7f5394009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=56098a49c087363e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:45.065 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.478+0000 7f538ffff700 1 -- 172.21.15.90:0/4064800183 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5394004160 con 0x7f53981054f0 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.478+0000 7f538ffff700 1 -- 172.21.15.90:0/4064800183 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f53940042c0 con 0x7f53981054f0 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.478+0000 7f538ffff700 1 -- 172.21.15.90:0/4064800183 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5394011a00 con 0x7f53981054f0 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- 172.21.15.90:0/4064800183 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 msgr2=0x7f5398105910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 --2- 172.21.15.90:0/4064800183 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f53940098a0 tx=0x7f5394009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- 172.21.15.90:0/4064800183 shutdown_connections 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 --2- 172.21.15.90:0/4064800183 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f5398105910 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- 172.21.15.90:0/4064800183 >> 172.21.15.90:0/4064800183 conn(0x7f5398100ad0 msgr2=0x7f5398102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- 172.21.15.90:0/4064800183 shutdown_connections 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- 172.21.15.90:0/4064800183 wait complete. 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 Processor -- start 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.479+0000 7f539f79d700 1 -- start start 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539f79d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539f79d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f539819a9f0 con 0x7f53981054f0 2024-04-06T13:58:45.066 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539d539700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539d539700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49100/0 (socket says 172.21.15.90:49100) 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539d539700 1 -- 172.21.15.90:0/4105395470 learned_addr learned my addr 172.21.15.90:0/4105395470 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539d539700 1 -- 172.21.15.90:0/4105395470 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5394009550 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539d539700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f5394009fb0 tx=0x7f5394011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5394003a40 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5394003ba0 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f539401a750 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f539819abf0 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.480+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f539819b700 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.481+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 7) v1 ==== 45435+0+0 (secure 0 0 0) 0x7f5394024430 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.481+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f539819b090 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.481+0000 7f538e7fc700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 0x7f538403aa50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.482+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f539404c530 con 0x7f53981054f0 2024-04-06T13:58:45.067 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.482+0000 7f539cd38700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 0x7f538403aa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.482+0000 7f539cd38700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 0x7f538403aa50 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f5388006fd0 tx=0x7f5388006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.487+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f539819b090 con 0x7f53981054f0 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.700+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) v1 -- 0x7f539819b090 con 0x7f5384038590 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.709+0000 7f538e7fc700 1 -- 172.21.15.90:0/4105395470 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f539819b090 con 0x7f5384038590 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 msgr2=0x7f538403aa50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 0x7f538403aa50 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f5388006fd0 tx=0x7f5388006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 msgr2=0x7f539819a4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f5394009fb0 tx=0x7f5394011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 shutdown_connections 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f5384038590 0x7f538403aa50 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 --2- 172.21.15.90:0/4105395470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53981054f0 0x7f539819a4b0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 >> 172.21.15.90:0/4105395470 conn(0x7f5398100ad0 msgr2=0x7f5398101a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 shutdown_connections 2024-04-06T13:58:45.068 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:44.711+0000 7f539f79d700 1 -- 172.21.15.90:0/4105395470 wait complete. 2024-04-06T13:58:45.929 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: [06/Apr/2024:13:58:44] ENGINE Bus STARTING 2024-04-06T13:58:45.929 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: [06/Apr/2024:13:58:44] ENGINE Serving on http://172.21.15.90:8765 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: [06/Apr/2024:13:58:44] ENGINE Serving on https://172.21.15.90:7150 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: [06/Apr/2024:13:58:44] ENGINE Bus STARTED 2024-04-06T13:58:45.930 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:45 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:58:46.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc00865b700 1 Processor -- start 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc00865b700 1 -- start start 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc00865b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc00865b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc000079e80 con 0x7fc0001070b0 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc0063f7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc0063f7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49114/0 (socket says 172.21.15.90:49114) 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.408+0000 7fc0063f7700 1 -- 172.21.15.90:0/1988135093 learned_addr learned my addr 172.21.15.90:0/1988135093 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc0063f7700 1 -- 172.21.15.90:0/1988135093 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0001099e0 con 0x7fc0001070b0 2024-04-06T13:58:46.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc0063f7700 1 --2- 172.21.15.90:0/1988135093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fbffc0098a0 tx=0x7fbffc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5d3f243243c97ff server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc0053f5700 1 -- 172.21.15.90:0/1988135093 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbffc004160 con 0x7fc0001070b0 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc0053f5700 1 -- 172.21.15.90:0/1988135093 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbffc0042c0 con 0x7fc0001070b0 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc0053f5700 1 -- 172.21.15.90:0/1988135093 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbffc011a00 con 0x7fc0001070b0 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc00865b700 1 -- 172.21.15.90:0/1988135093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 msgr2=0x7fc0001094a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.409+0000 7fc00865b700 1 --2- 172.21.15.90:0/1988135093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fbffc0098a0 tx=0x7fbffc009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- 172.21.15.90:0/1988135093 shutdown_connections 2024-04-06T13:58:46.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 --2- 172.21.15.90:0/1988135093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc0001094a0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- 172.21.15.90:0/1988135093 >> 172.21.15.90:0/1988135093 conn(0x7fc000100aa0 msgr2=0x7fc000102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- 172.21.15.90:0/1988135093 shutdown_connections 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- 172.21.15.90:0/1988135093 wait complete. 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 Processor -- start 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- start start 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc00865b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc00019a9f0 con 0x7fc0001070b0 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc0063f7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc0063f7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49122/0 (socket says 172.21.15.90:49122) 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.410+0000 7fc0063f7700 1 -- 172.21.15.90:0/4155472854 learned_addr learned my addr 172.21.15.90:0/4155472854 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fc0063f7700 1 -- 172.21.15.90:0/4155472854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbffc009550 con 0x7fc0001070b0 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fc0063f7700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fbffc009130 tx=0x7fbffc011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:46.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbffc0038b0 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbffc003a10 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbffc019550 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc00019abf0 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.411+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc00019b750 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.412+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 7) v1 ==== 45435+0+0 (secure 0 0 0) 0x7fbffc01a430 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.412+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc000104790 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.412+0000 7fbff77fe700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 0x7fbfec03aa80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.412+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fbffc04c560 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.412+0000 7fc005bf6700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 0x7fbfec03aa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.413+0000 7fc005bf6700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 0x7fbfec03aa80 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006fd0 tx=0x7fbff0006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.417+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbffc023840 con 0x7fc0001070b0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.608+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) v1 -- 0x7fc000101e20 con 0x7fbfec0385c0 2024-04-06T13:58:46.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.609+0000 7fbff77fe700 1 -- 172.21.15.90:0/4155472854 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+16 (secure 0 0 0) 0x7fc000101e20 con 0x7fbfec0385c0 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 msgr2=0x7fbfec03aa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 0x7fbfec03aa80 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006fd0 tx=0x7fbff0006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 msgr2=0x7fc00019a4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fbffc009130 tx=0x7fbffc011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 shutdown_connections 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fbfec0385c0 0x7fbfec03aa80 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 --2- 172.21.15.90:0/4155472854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0001070b0 0x7fc00019a4b0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 >> 172.21.15.90:0/4155472854 conn(0x7fc000100aa0 msgr2=0x7fc0001016f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 shutdown_connections 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:45.611+0000 7fc00865b700 1 -- 172.21.15.90:0/4155472854 wait complete. 2024-04-06T13:58:46.012 INFO:teuthology.orchestra.run.smithi090.stdout:Generating ssh key... 2024-04-06T13:58:46.954 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-mon[43420]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:46.954 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-mon[43420]: mgrmap e8: x(active, since 2s) 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: Generating public/private rsa key pair. 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: Your identification has been saved in /tmp/tmpckcm_aw7/key. 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: Your public key has been saved in /tmp/tmpckcm_aw7/key.pub. 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: The key fingerprint is: 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: SHA256:Ug/XhwK4GE9PUEzJ9v1vI6wvXcFAMYD91wgbz7Oz6g4 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: The key's randomart image is: 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: +---[RSA 3072]----+ 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | .B+.o.o+. | 2024-04-06T13:58:46.955 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | . o *o ooo. | 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | = =o.o.+*+..| 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | . o..+..oo=+.| 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | . S . ..o.| 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | . +. | 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | E o .+ | 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | o +..o| 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: | +Bo...| 2024-04-06T13:58:46.956 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: +----[SHA256]-----+ 2024-04-06T13:58:47.032 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc0d5d7700 1 Processor -- start 2024-04-06T13:58:47.032 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc0d5d7700 1 -- start start 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc0d5d7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc0d5d7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc08105320 con 0x7efc08104930 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc06ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc06ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49126/0 (socket says 172.21.15.90:49126) 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc06ffd700 1 -- 172.21.15.90:0/1951787491 learned_addr learned my addr 172.21.15.90:0/1951787491 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc06ffd700 1 -- 172.21.15.90:0/1951787491 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc081078c0 con 0x7efc08104930 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc06ffd700 1 --2- 172.21.15.90:0/1951787491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7efbf80098a0 tx=0x7efbf8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a95148cd89c2eec5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.323+0000 7efc05ffb700 1 -- 172.21.15.90:0/1951787491 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efbf8004160 con 0x7efc08104930 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc05ffb700 1 -- 172.21.15.90:0/1951787491 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efbf80042c0 con 0x7efc08104930 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc05ffb700 1 -- 172.21.15.90:0/1951787491 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efbf8011a00 con 0x7efc08104930 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 -- 172.21.15.90:0/1951787491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 msgr2=0x7efc08104d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/1951787491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7efbf80098a0 tx=0x7efbf8009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 -- 172.21.15.90:0/1951787491 shutdown_connections 2024-04-06T13:58:47.033 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/1951787491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc08104d50 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 -- 172.21.15.90:0/1951787491 >> 172.21.15.90:0/1951787491 conn(0x7efc081002c0 msgr2=0x7efc081026e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 -- 172.21.15.90:0/1951787491 shutdown_connections 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.324+0000 7efc0d5d7700 1 -- 172.21.15.90:0/1951787491 wait complete. 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc0d5d7700 1 Processor -- start 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc0d5d7700 1 -- start start 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc0d5d7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc0d5d7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc08105870 con 0x7efc08104930 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc06ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc06ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49128/0 (socket says 172.21.15.90:49128) 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.325+0000 7efc06ffd700 1 -- 172.21.15.90:0/3611277173 learned_addr learned my addr 172.21.15.90:0/3611277173 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efc06ffd700 1 -- 172.21.15.90:0/3611277173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efbf8009550 con 0x7efc08104930 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efc06ffd700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7efbf8009130 tx=0x7efbf8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efbf8003ab0 con 0x7efc08104930 2024-04-06T13:58:47.034 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc08105a70 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efbf8003c10 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efbf80247d0 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.326+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc08105f10 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7efbf802d3f0 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efbeffff700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 0x7efbf003abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7efbf804c840 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc08103f50 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efc067fc700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 0x7efbf003abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.327+0000 7efc067fc700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 0x7efbf003abc0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7efbfc006fd0 tx=0x7efbfc006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.332+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efbf8048080 con 0x7efc08104930 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.522+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) v1 -- 0x7efc08106260 con 0x7efbf0038700 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.672+0000 7efbeffff700 1 -- 172.21.15.90:0/3611277173 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7efc08106260 con 0x7efbf0038700 2024-04-06T13:58:47.035 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 msgr2=0x7efbf003abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 0x7efbf003abc0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7efbfc006fd0 tx=0x7efbfc006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 msgr2=0x7efc081072d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7efbf8009130 tx=0x7efbf8011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 shutdown_connections 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7efbf0038700 0x7efbf003abc0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 --2- 172.21.15.90:0/3611277173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08104930 0x7efc081072d0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 >> 172.21.15.90:0/3611277173 conn(0x7efc081002c0 msgr2=0x7efc08100ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 shutdown_connections 2024-04-06T13:58:47.036 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:46.675+0000 7efc0d5d7700 1 -- 172.21.15.90:0/3611277173 wait complete. 2024-04-06T13:58:47.879 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:47 smithi090 ceph-mon[43420]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:47.879 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:47 smithi090 ceph-mon[43420]: Generating ssh key... 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2ebbe2700 1 Processor -- start 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2ebbe2700 1 -- start start 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2ebbe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2ebbe2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2e4079ec0 con 0x7ff2e4107070 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2e997e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2e997e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49144/0 (socket says 172.21.15.90:49144) 2024-04-06T13:58:47.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2e997e700 1 -- 172.21.15.90:0/287430721 learned_addr learned my addr 172.21.15.90:0/287430721 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.364+0000 7ff2e997e700 1 -- 172.21.15.90:0/287430721 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff2e41099a0 con 0x7ff2e4107070 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2e997e700 1 --2- 172.21.15.90:0/287430721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7ff2e00098a0 tx=0x7ff2e0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9f446aee0c84773 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2e897c700 1 -- 172.21.15.90:0/287430721 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff2e0004160 con 0x7ff2e4107070 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2e897c700 1 -- 172.21.15.90:0/287430721 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff2e00042c0 con 0x7ff2e4107070 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2e897c700 1 -- 172.21.15.90:0/287430721 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff2e0011a00 con 0x7ff2e4107070 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/287430721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 msgr2=0x7ff2e4109460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/287430721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7ff2e00098a0 tx=0x7ff2e0009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/287430721 shutdown_connections 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/287430721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4109460 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/287430721 >> 172.21.15.90:0/287430721 conn(0x7ff2e4100aa0 msgr2=0x7ff2e4102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/287430721 shutdown_connections 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.365+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/287430721 wait complete. 2024-04-06T13:58:47.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2ebbe2700 1 Processor -- start 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2ebbe2700 1 -- start start 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2ebbe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2ebbe2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2e4105020 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2e997e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2e997e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49146/0 (socket says 172.21.15.90:49146) 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.366+0000 7ff2e997e700 1 -- 172.21.15.90:0/1561406403 learned_addr learned my addr 172.21.15.90:0/1561406403 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2e997e700 1 -- 172.21.15.90:0/1561406403 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff2e0009550 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2e997e700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7ff2e0009130 tx=0x7ff2e0011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff2e00038b0 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff2e0003a10 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff2e0019550 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff2e4105220 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.367+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff2e41056c0 con 0x7ff2e4107070 2024-04-06T13:58:47.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7ff2e0019810 con 0x7ff2e4107070 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff2e4075420 con 0x7ff2e4107070 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2daffd700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 0x7ff2d003ab50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7ff2e004c910 con 0x7ff2e4107070 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2e917d700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 0x7ff2d003ab50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.368+0000 7ff2e917d700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 0x7ff2d003ab50 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ff2d4006fd0 tx=0x7ff2d4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.373+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff2e0018d20 con 0x7ff2e4107070 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.566+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) v1 -- 0x7ff2e4105b00 con 0x7ff2d0038690 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.566+0000 7ff2daffd700 1 -- 172.21.15.90:0/1561406403 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+595 (secure 0 0 0) 0x7ff2e4105b00 con 0x7ff2d0038690 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 msgr2=0x7ff2d003ab50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 0x7ff2d003ab50 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7ff2d4006fd0 tx=0x7ff2d4006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 msgr2=0x7ff2e4106a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7ff2e0009130 tx=0x7ff2e0011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 shutdown_connections 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff2d0038690 0x7ff2d003ab50 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.568+0000 7ff2ebbe2700 1 --2- 172.21.15.90:0/1561406403 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2e4107070 0x7ff2e4106a80 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.569+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 >> 172.21.15.90:0/1561406403 conn(0x7ff2e4100aa0 msgr2=0x7ff2e41016d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.569+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 shutdown_connections 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:47.569+0000 7ff2ebbe2700 1 -- 172.21.15.90:0/1561406403 wait complete. 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:Adding key to root@localhost authorized_keys... 2024-04-06T13:58:47.963 INFO:teuthology.orchestra.run.smithi090.stdout:Adding host smithi090... 2024-04-06T13:58:48.989 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:48 smithi090 ceph-mon[43420]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:49.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:49 smithi090 ceph-mon[43420]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi090", "addr": "172.21.15.90", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:50 smithi090 ceph-mon[43420]: Deploying cephadm binary to smithi090 2024-04-06T13:58:51.227 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Added host 'smithi090' with addr '172.21.15.90' 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab0b97b700 1 Processor -- start 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab0b97b700 1 -- start start 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab0b97b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab0b97b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab04079e60 con 0x7fab041070d0 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab09717700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab09717700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49148/0 (socket says 172.21.15.90:49148) 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab09717700 1 -- 172.21.15.90:0/3471220138 learned_addr learned my addr 172.21.15.90:0/3471220138 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:51.228 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.527+0000 7fab09717700 1 -- 172.21.15.90:0/3471220138 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab04109a00 con 0x7fab041070d0 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab09717700 1 --2- 172.21.15.90:0/3471220138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fab000098a0 tx=0x7fab00009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1db4c92c196712f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7faafbfff700 1 -- 172.21.15.90:0/3471220138 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fab00004160 con 0x7fab041070d0 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7faafbfff700 1 -- 172.21.15.90:0/3471220138 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fab000042c0 con 0x7fab041070d0 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7faafbfff700 1 -- 172.21.15.90:0/3471220138 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fab00011a00 con 0x7fab041070d0 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 -- 172.21.15.90:0/3471220138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 msgr2=0x7fab041094c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3471220138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7fab000098a0 tx=0x7fab00009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 -- 172.21.15.90:0/3471220138 shutdown_connections 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3471220138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab041094c0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 -- 172.21.15.90:0/3471220138 >> 172.21.15.90:0/3471220138 conn(0x7fab04100aa0 msgr2=0x7fab04102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 -- 172.21.15.90:0/3471220138 shutdown_connections 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.528+0000 7fab0b97b700 1 -- 172.21.15.90:0/3471220138 wait complete. 2024-04-06T13:58:51.229 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab0b97b700 1 Processor -- start 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab0b97b700 1 -- start start 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab0b97b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab0b97b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab04105080 con 0x7fab041070d0 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab09717700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab09717700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49156/0 (socket says 172.21.15.90:49156) 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab09717700 1 -- 172.21.15.90:0/3733475764 learned_addr learned my addr 172.21.15.90:0/3733475764 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.529+0000 7fab09717700 1 -- 172.21.15.90:0/3733475764 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab00009550 con 0x7fab041070d0 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7fab09717700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fab00009fb0 tx=0x7fab00011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:51.230 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fab00003a40 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fab00003ba0 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fab0001a750 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab04105280 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.530+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab04105720 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7fab0001a980 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab04075420 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7faafa7fc700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 0x7faaf003abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fab0004c840 con 0x7fab041070d0 2024-04-06T13:58:51.231 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7fab08f16700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 0x7faaf003abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.531+0000 7fab08f16700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 0x7faaf003abc0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7faaf4006fd0 tx=0x7faaf4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.536+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fab00018db0 con 0x7fab041070d0 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:48.732+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi090", "addr": "172.21.15.90", "target": ["mon-mgr", ""]}) v1 -- 0x7fab04105b20 con 0x7faaf0038700 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.910+0000 7faafa7fc700 1 -- 172.21.15.90:0/3733475764 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+48 (secure 0 0 0) 0x7fab04105b20 con 0x7faaf0038700 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 msgr2=0x7faaf003abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 0x7faaf003abc0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7faaf4006fd0 tx=0x7faaf4006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 msgr2=0x7fab04106ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fab00009fb0 tx=0x7fab00011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 shutdown_connections 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7faaf0038700 0x7faaf003abc0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:51.232 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 --2- 172.21.15.90:0/3733475764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab041070d0 0x7fab04106ae0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:51.233 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 >> 172.21.15.90:0/3733475764 conn(0x7fab04100aa0 msgr2=0x7fab04101650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:51.233 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 shutdown_connections 2024-04-06T13:58:51.233 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:50.913+0000 7fab0b97b700 1 -- 172.21.15.90:0/3733475764 wait complete. 2024-04-06T13:58:51.233 INFO:teuthology.orchestra.run.smithi090.stdout:Deploying unmanaged mon service... 2024-04-06T13:58:52.070 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:51 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:52.071 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:51 smithi090 ceph-mon[43420]: Added host smithi090 2024-04-06T13:58:52.071 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:51 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:58:52.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-06T13:58:52.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.688+0000 7ff9cb518700 1 Processor -- start 2024-04-06T13:58:52.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.688+0000 7ff9cb518700 1 -- start start 2024-04-06T13:58:52.399 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9cb518700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9cb518700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9c4079e80 con 0x7ff9c4104ea0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9c92b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9c92b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49162/0 (socket says 172.21.15.90:49162) 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9c92b4700 1 -- 172.21.15.90:0/1203602448 learned_addr learned my addr 172.21.15.90:0/1203602448 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.689+0000 7ff9c92b4700 1 -- 172.21.15.90:0/1203602448 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9c4105800 con 0x7ff9c4104ea0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9c92b4700 1 --2- 172.21.15.90:0/1203602448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff9c00098a0 tx=0x7ff9c0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2dcab31a763f076e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9bbfff700 1 -- 172.21.15.90:0/1203602448 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff9c0004160 con 0x7ff9c4104ea0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9bbfff700 1 -- 172.21.15.90:0/1203602448 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff9c00042c0 con 0x7ff9c4104ea0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9bbfff700 1 -- 172.21.15.90:0/1203602448 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff9c0011a00 con 0x7ff9c4104ea0 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9cb518700 1 -- 172.21.15.90:0/1203602448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 msgr2=0x7ff9c41052c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.690+0000 7ff9cb518700 1 --2- 172.21.15.90:0/1203602448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff9c00098a0 tx=0x7ff9c0009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- 172.21.15.90:0/1203602448 shutdown_connections 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 --2- 172.21.15.90:0/1203602448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c41052c0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:52.400 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- 172.21.15.90:0/1203602448 >> 172.21.15.90:0/1203602448 conn(0x7ff9c4100aa0 msgr2=0x7ff9c4102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- 172.21.15.90:0/1203602448 shutdown_connections 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- 172.21.15.90:0/1203602448 wait complete. 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 Processor -- start 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- start start 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9cb518700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9c419cb40 con 0x7ff9c4104ea0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.691+0000 7ff9c92b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9c92b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49164/0 (socket says 172.21.15.90:49164) 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9c92b4700 1 -- 172.21.15.90:0/3683627730 learned_addr learned my addr 172.21.15.90:0/3683627730 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9c92b4700 1 -- 172.21.15.90:0/3683627730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9c0009550 con 0x7ff9c4104ea0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9c92b4700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ff9c0009fb0 tx=0x7ff9c0011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff9c0003a40 con 0x7ff9c4104ea0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff9c419cd40 con 0x7ff9c4104ea0 2024-04-06T13:58:52.401 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff9c0003ba0 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7ff9c001a750 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.692+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9c419d8a0 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.693+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7ff9c001a980 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.693+0000 7ff9ba7fc700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 0x7ff9b003abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.693+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7ff9c004c7a0 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.693+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff9c4104790 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.693+0000 7ff9c8ab3700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 0x7ff9b003abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.694+0000 7ff9c8ab3700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 0x7ff9b003abc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7ff9b4006fd0 tx=0x7ff9b4006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.698+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff9c001ac30 con 0x7ff9c4104ea0 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.914+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7ff9c419dae0 con 0x7ff9b0038700 2024-04-06T13:58:52.402 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.919+0000 7ff9ba7fc700 1 -- 172.21.15.90:0/3683627730 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7ff9c419dae0 con 0x7ff9b0038700 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 msgr2=0x7ff9b003abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 0x7ff9b003abc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7ff9b4006fd0 tx=0x7ff9b4006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 msgr2=0x7ff9c419c600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ff9c0009fb0 tx=0x7ff9c0011ad0 comp rx=0 tx=0).stop 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 shutdown_connections 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7ff9b0038700 0x7ff9b003abc0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 --2- 172.21.15.90:0/3683627730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff9c4104ea0 0x7ff9c419c600 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 >> 172.21.15.90:0/3683627730 conn(0x7ff9c4100aa0 msgr2=0x7ff9c41933b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 shutdown_connections 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:51.921+0000 7ff9cb518700 1 -- 172.21.15.90:0/3683627730 wait complete. 2024-04-06T13:58:52.403 INFO:teuthology.orchestra.run.smithi090.stdout:Deploying unmanaged mgr service... 2024-04-06T13:58:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:52 smithi090 ceph-mon[43420]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:53.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:52 smithi090 ceph-mon[43420]: Saving service mon spec with placement count:5 2024-04-06T13:58:53.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:52 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:53.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:52 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:53.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:52 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:53.553 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-06T13:58:53.553 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.740+0000 7fd86110d700 1 Processor -- start 2024-04-06T13:58:53.553 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.740+0000 7fd86110d700 1 -- start start 2024-04-06T13:58:53.553 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.740+0000 7fd86110d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.740+0000 7fd86110d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd85c079e40 con 0x7fd85c1070e0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd85ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd85ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49178/0 (socket says 172.21.15.90:49178) 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd85ad9d700 1 -- 172.21.15.90:0/327133745 learned_addr learned my addr 172.21.15.90:0/327133745 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd85ad9d700 1 -- 172.21.15.90:0/327133745 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd85c109a10 con 0x7fd85c1070e0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd85ad9d700 1 --2- 172.21.15.90:0/327133745 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd84c0098a0 tx=0x7fd84c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b5376bf43e461940 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd859d9b700 1 -- 172.21.15.90:0/327133745 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd84c004160 con 0x7fd85c1070e0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd859d9b700 1 -- 172.21.15.90:0/327133745 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd84c0042c0 con 0x7fd85c1070e0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.741+0000 7fd859d9b700 1 -- 172.21.15.90:0/327133745 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd84c0119b0 con 0x7fd85c1070e0 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 -- 172.21.15.90:0/327133745 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 msgr2=0x7fd85c1094d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 --2- 172.21.15.90:0/327133745 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd84c0098a0 tx=0x7fd84c009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 -- 172.21.15.90:0/327133745 shutdown_connections 2024-04-06T13:58:53.554 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 --2- 172.21.15.90:0/327133745 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c1094d0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 -- 172.21.15.90:0/327133745 >> 172.21.15.90:0/327133745 conn(0x7fd85c100ad0 msgr2=0x7fd85c102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 -- 172.21.15.90:0/327133745 shutdown_connections 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.742+0000 7fd86110d700 1 -- 172.21.15.90:0/327133745 wait complete. 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd86110d700 1 Processor -- start 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd86110d700 1 -- start start 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd86110d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd86110d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd85c105090 con 0x7fd85c1070e0 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd85ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd85ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:49188/0 (socket says 172.21.15.90:49188) 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd85ad9d700 1 -- 172.21.15.90:0/2749599829 learned_addr learned my addr 172.21.15.90:0/2749599829 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.743+0000 7fd85ad9d700 1 -- 172.21.15.90:0/2749599829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd84c009550 con 0x7fd85c1070e0 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd85ad9d700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fd84c011670 tx=0x7fd84c011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:53.555 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd84c003b20 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd84c003c80 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd85c105290 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd84c019770 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.744+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd85c105730 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7fd84c019a30 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd843fff700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 0x7fd84403abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fd84c04c950 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd85a59c700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 0x7fd84403abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd85c104780 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.745+0000 7fd85a59c700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 0x7fd84403abc0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fd850006fd0 tx=0x7fd850006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:52.750+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd84c018b80 con 0x7fd85c1070e0 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.016+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7fd85c105970 con 0x7fd844038700 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.021+0000 7fd843fff700 1 -- 172.21.15.90:0/2749599829 <== mgr.14120 v2:172.21.15.90:6800/2246618981 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fd85c105970 con 0x7fd844038700 2024-04-06T13:58:53.556 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 msgr2=0x7fd84403abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 0x7fd84403abc0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fd850006fd0 tx=0x7fd850006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 msgr2=0x7fd85c106af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fd84c011670 tx=0x7fd84c011a30 comp rx=0 tx=0).stop 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 shutdown_connections 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fd844038700 0x7fd84403abc0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 --2- 172.21.15.90:0/2749599829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd85c1070e0 0x7fd85c106af0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 >> 172.21.15.90:0/2749599829 conn(0x7fd85c100ad0 msgr2=0x7fd85c101610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 shutdown_connections 2024-04-06T13:58:53.557 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:53.025+0000 7fd86110d700 1 -- 172.21.15.90:0/2749599829 wait complete. 2024-04-06T13:58:54.184 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:54 smithi090 ceph-mon[43420]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:58:54.184 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:54 smithi090 ceph-mon[43420]: Saving service mgr spec with placement count:2 2024-04-06T13:58:54.184 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:54 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:54.184 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:54 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:54.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.029+0000 7f1d3ff3c700 1 Processor -- start 2024-04-06T13:58:54.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3ff3c700 1 -- start start 2024-04-06T13:58:54.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3ff3c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:54.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3ff3c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d381074f0 con 0x7f1d38106b00 2024-04-06T13:58:54.599 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3dcd8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3dcd8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54488/0 (socket says 172.21.15.90:54488) 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3dcd8700 1 -- 172.21.15.90:0/1638220118 learned_addr learned my addr 172.21.15.90:0/1638220118 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.030+0000 7f1d3dcd8700 1 -- 172.21.15.90:0/1638220118 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d38105710 con 0x7f1d38106b00 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3dcd8700 1 --2- 172.21.15.90:0/1638220118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f1d300098a0 tx=0x7f1d30009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e5d267b419e6d3e8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ccd6700 1 -- 172.21.15.90:0/1638220118 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1d30004160 con 0x7f1d38106b00 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ccd6700 1 -- 172.21.15.90:0/1638220118 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d300042c0 con 0x7f1d38106b00 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ccd6700 1 -- 172.21.15.90:0/1638220118 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1d300119b0 con 0x7f1d38106b00 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1638220118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 msgr2=0x7f1d38106f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1638220118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f1d300098a0 tx=0x7f1d30009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1638220118 shutdown_connections 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1638220118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38106f20 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1638220118 >> 172.21.15.90:0/1638220118 conn(0x7f1d38100200 msgr2=0x7f1d38102680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:54.600 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1638220118 shutdown_connections 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.031+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1638220118 wait complete. 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3ff3c700 1 Processor -- start 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3ff3c700 1 -- start start 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3ff3c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3ff3c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d3818edc0 con 0x7f1d38106b00 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3dcd8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3dcd8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54502/0 (socket says 172.21.15.90:54502) 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3dcd8700 1 -- 172.21.15.90:0/1318391304 learned_addr learned my addr 172.21.15.90:0/1318391304 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.032+0000 7f1d3dcd8700 1 -- 172.21.15.90:0/1318391304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d30009550 con 0x7f1d38106b00 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d3dcd8700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f1d30003e60 tx=0x7f1d30011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1d30003a60 con 0x7f1d38106b00 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d30003bc0 con 0x7f1d38106b00 2024-04-06T13:58:54.601 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1d300196c0 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d3818efc0 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.033+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d3818f460 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7f1d30019930 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d381945c0 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d36ffd700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 0x7f1d2003abf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f1d3004cab0 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d3d4d7700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 0x7f1d2003abf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.034+0000 7f1d3d4d7700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 0x7f1d2003abf0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f1d2c006fd0 tx=0x7f1d2c006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.038+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f1d30010bb0 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.232+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) v1 -- 0x7f1d38075420 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.239+0000 7f1d36ffd700 1 -- 172.21.15.90:0/1318391304 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) v1 ==== 142+0+0 (secure 0 0 0) 0x7f1d30010380 con 0x7f1d38106b00 2024-04-06T13:58:54.602 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 msgr2=0x7f1d2003abf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 0x7f1d2003abf0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f1d2c006fd0 tx=0x7f1d2c006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 msgr2=0x7f1d38190820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f1d30003e60 tx=0x7f1d30011a30 comp rx=0 tx=0).stop 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 shutdown_connections 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f1d20038730 0x7f1d2003abf0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 --2- 172.21.15.90:0/1318391304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d38106b00 0x7f1d38190820 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 >> 172.21.15.90:0/1318391304 conn(0x7f1d38100200 msgr2=0x7f1d381011e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 shutdown_connections 2024-04-06T13:58:54.603 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:54.241+0000 7f1d3ff3c700 1 -- 172.21.15.90:0/1318391304 wait complete. 2024-04-06T13:58:55.517 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:55 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1318391304' entity='client.admin' 2024-04-06T13:58:55.517 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:55 smithi090 ceph-mon[43420]: from='mgr.14120 172.21.15.90:0/2101250948' entity='mgr.x' 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.078+0000 7f97a6307700 1 Processor -- start 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.078+0000 7f97a6307700 1 -- start start 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f97a6307700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f97a6307700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a0105ee0 con 0x7f97a01054f0 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:55.781 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54512/0 (socket says 172.21.15.90:54512) 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979ffff700 1 -- 172.21.15.90:0/1321864622 learned_addr learned my addr 172.21.15.90:0/1321864622 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979ffff700 1 -- 172.21.15.90:0/1321864622 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97a01066f0 con 0x7f97a01054f0 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979ffff700 1 --2- 172.21.15.90:0/1321864622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f979400bb40 tx=0x7f979400be50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f2d453b258385451 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979effd700 1 -- 172.21.15.90:0/1321864622 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f979400dbe0 con 0x7f97a01054f0 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979effd700 1 -- 172.21.15.90:0/1321864622 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f97940044f0 con 0x7f97a01054f0 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.079+0000 7f979effd700 1 -- 172.21.15.90:0/1321864622 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f9794003e60 con 0x7f97a01054f0 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 -- 172.21.15.90:0/1321864622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 msgr2=0x7f97a0105910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 --2- 172.21.15.90:0/1321864622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f979400bb40 tx=0x7f979400be50 comp rx=0 tx=0).stop 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 -- 172.21.15.90:0/1321864622 shutdown_connections 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 --2- 172.21.15.90:0/1321864622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0105910 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 -- 172.21.15.90:0/1321864622 >> 172.21.15.90:0/1321864622 conn(0x7f97a0100ad0 msgr2=0x7f97a0102ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 -- 172.21.15.90:0/1321864622 shutdown_connections 2024-04-06T13:58:55.782 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.080+0000 7f97a6307700 1 -- 172.21.15.90:0/1321864622 wait complete. 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f97a6307700 1 Processor -- start 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f97a6307700 1 -- start start 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f97a6307700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f97a6307700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97a01987a0 con 0x7f97a01054f0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f979ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f979ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54518/0 (socket says 172.21.15.90:54518) 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.081+0000 7f979ffff700 1 -- 172.21.15.90:0/55944734 learned_addr learned my addr 172.21.15.90:0/55944734 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f979ffff700 1 -- 172.21.15.90:0/55944734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f979400b7f0 con 0x7f97a01054f0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f979ffff700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f9794003bf0 tx=0x7f9794003fb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f979401e070 con 0x7f97a01054f0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97a01989a0 con 0x7f97a01054f0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f97940109c0 con 0x7f97a01054f0 2024-04-06T13:58:55.783 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f97940229a0 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.082+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97a0199500 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7f979402c440 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f979d7fa700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 0x7f978803abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f979404eab0 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f979f7fe700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 0x7f978803abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97a0198e40 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.083+0000 7f979f7fe700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 0x7f978803abc0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f9790006fd0 tx=0x7f9790006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.088+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f97a0198e40 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.295+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) v1 -- 0x7f97a0198e40 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.301+0000 7f979d7fa700 1 -- 172.21.15.90:0/55944734 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) v1 ==== 130+0+0 (secure 0 0 0) 0x7f9794027070 con 0x7f97a01054f0 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.303+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 msgr2=0x7f978803abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 0x7f978803abc0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f9790006fd0 tx=0x7f9790006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 msgr2=0x7f97a0198260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:55.784 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f9794003bf0 tx=0x7f9794003fb0 comp rx=0 tx=0).stop 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 shutdown_connections 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f9788038700 0x7f978803abc0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 --2- 172.21.15.90:0/55944734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97a01054f0 0x7f97a0198260 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 >> 172.21.15.90:0/55944734 conn(0x7f97a0100ad0 msgr2=0x7f97a018eec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 shutdown_connections 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:55.304+0000 7f97a6307700 1 -- 172.21.15.90:0/55944734 wait complete. 2024-04-06T13:58:55.785 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling the dashboard module... 2024-04-06T13:58:56.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:56 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/55944734' entity='client.admin' 2024-04-06T13:58:57.687 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4192270146' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-06T13:58:57.688 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setuser ceph since I am not root 2024-04-06T13:58:57.688 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setgroup ceph since I am not root 2024-04-06T13:58:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.125+0000 7f06d2ab2700 1 Processor -- start 2024-04-06T13:58:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.125+0000 7f06d2ab2700 1 -- start start 2024-04-06T13:58:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.125+0000 7f06d2ab2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.125+0000 7f06d2ab2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06cc079e40 con 0x7f06cc107110 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06d084e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06d084e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54524/0 (socket says 172.21.15.90:54524) 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06d084e700 1 -- 172.21.15.90:0/2480753255 learned_addr learned my addr 172.21.15.90:0/2480753255 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06d084e700 1 -- 172.21.15.90:0/2480753255 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06cc109a40 con 0x7f06cc107110 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06d084e700 1 --2- 172.21.15.90:0/2480753255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f06c00098a0 tx=0x7f06c0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=11340274164a64d4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06cb7fe700 1 -- 172.21.15.90:0/2480753255 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f06c0004160 con 0x7f06cc107110 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06cb7fe700 1 -- 172.21.15.90:0/2480753255 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06c00042c0 con 0x7f06cc107110 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.126+0000 7f06cb7fe700 1 -- 172.21.15.90:0/2480753255 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f06c00119b0 con 0x7f06cc107110 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- 172.21.15.90:0/2480753255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 msgr2=0x7f06cc109500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/2480753255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f06c00098a0 tx=0x7f06c0009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- 172.21.15.90:0/2480753255 shutdown_connections 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/2480753255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc109500 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- 172.21.15.90:0/2480753255 >> 172.21.15.90:0/2480753255 conn(0x7f06cc100b00 msgr2=0x7f06cc102f20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- 172.21.15.90:0/2480753255 shutdown_connections 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- 172.21.15.90:0/2480753255 wait complete. 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 Processor -- start 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- start start 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.127+0000 7f06d2ab2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06cc1050c0 con 0x7f06cc107110 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d084e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d084e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54526/0 (socket says 172.21.15.90:54526) 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d084e700 1 -- 172.21.15.90:0/4192270146 learned_addr learned my addr 172.21.15.90:0/4192270146 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d084e700 1 -- 172.21.15.90:0/4192270146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06c0009550 con 0x7f06cc107110 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d084e700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f06c0006af0 tx=0x7f06c0011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f06c0003a40 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06c0003ba0 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f06cc1052c0 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f06c00196c0 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.128+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06cc105760 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.129+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 8) v1 ==== 45541+0+0 (secure 0 0 0) 0x7f06c0019980 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.129+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f06cc075420 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.130+0000 7f06c9ffb700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 0x7f06bc03ab70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.130+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f06c004c800 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.130+0000 7f06cbfff700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 0x7f06bc03ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.130+0000 7f06cbfff700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 0x7f06bc03ab70 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f06b8006fd0 tx=0x7f06b8006e40 comp rx=0 tx=0).ready entity=mgr.14120 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.134+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f06c0018760 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:56.383+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) v1 -- 0x7f06cc10a7d0 con 0x7f06cc107110 2024-04-06T13:58:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.310+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 9) v1 ==== 45554+0+0 (secure 0 0 0) 0x7f06c002dcd0 con 0x7f06cc107110 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.311+0000 7f06c9ffb700 1 -- 172.21.15.90:0/4192270146 <== mon.0 v2:172.21.15.90:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) v1 ==== 88+0+0 (secure 0 0 0) 0x7f06c000b840 con 0x7f06cc107110 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 msgr2=0x7f06bc03ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 0x7f06bc03ab70 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f06b8006fd0 tx=0x7f06b8006e40 comp rx=0 tx=0).stop 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 msgr2=0x7f06cc106ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f06c0006af0 tx=0x7f06c0011a10 comp rx=0 tx=0).stop 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 shutdown_connections 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f06bc0386b0 0x7f06bc03ab70 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 --2- 172.21.15.90:0/4192270146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06cc107110 0x7f06cc106ad0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.314+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 >> 172.21.15.90:0/4192270146 conn(0x7f06cc100b00 msgr2=0x7f06cc101660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.315+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 shutdown_connections 2024-04-06T13:58:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:57.315+0000 7f06d2ab2700 1 -- 172.21.15.90:0/4192270146 wait complete. 2024-04-06T13:58:57.958 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:57.766+0000 7f6ac27fd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T13:58:57.958 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:57.862+0000 7f6ac27fd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T13:58:58.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:57.957+0000 7f6ac27fd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T13:58:58.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:58 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:58.045+0000 7f6ac27fd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T13:58:58.642 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:58 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4192270146' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-06T13:58:58.642 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:58 smithi090 ceph-mon[43420]: mgrmap e9: x(active, since 14s) 2024-04-06T13:58:58.799 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:58:58.799 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-06T13:58:58.799 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "active_name": "x", 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a9d80700 1 Processor -- start 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a9d80700 1 -- start start 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a9d80700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a9d80700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20a4079e80 con 0x7f20a41070b0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54544/0 (socket says 172.21.15.90:54544) 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.144+0000 7f20a37fe700 1 -- 172.21.15.90:0/2433060361 learned_addr learned my addr 172.21.15.90:0/2433060361 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.145+0000 7f20a37fe700 1 -- 172.21.15.90:0/2433060361 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20a41099e0 con 0x7f20a41070b0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.145+0000 7f20a37fe700 1 --2- 172.21.15.90:0/2433060361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f20940098a0 tx=0x7f2094009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f1620c56f2c43c8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.145+0000 7f20a27fc700 1 -- 172.21.15.90:0/2433060361 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2094004160 con 0x7f20a41070b0 2024-04-06T13:58:58.800 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.145+0000 7f20a27fc700 1 -- 172.21.15.90:0/2433060361 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f20940042c0 con 0x7f20a41070b0 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.145+0000 7f20a27fc700 1 -- 172.21.15.90:0/2433060361 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f20940119b0 con 0x7f20a41070b0 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.146+0000 7f20a9d80700 1 -- 172.21.15.90:0/2433060361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 msgr2=0x7f20a41094a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.146+0000 7f20a9d80700 1 --2- 172.21.15.90:0/2433060361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f20940098a0 tx=0x7f2094009bb0 comp rx=0 tx=0).stop 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 -- 172.21.15.90:0/2433060361 shutdown_connections 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 --2- 172.21.15.90:0/2433060361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a41094a0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 -- 172.21.15.90:0/2433060361 >> 172.21.15.90:0/2433060361 conn(0x7f20a4100aa0 msgr2=0x7f20a4102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 -- 172.21.15.90:0/2433060361 shutdown_connections 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 -- 172.21.15.90:0/2433060361 wait complete. 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 Processor -- start 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.147+0000 7f20a9d80700 1 -- start start 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a9d80700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a9d80700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20a419aa10 con 0x7f20a41070b0 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54548/0 (socket says 172.21.15.90:54548) 2024-04-06T13:58:58.801 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a37fe700 1 -- 172.21.15.90:0/1512111104 learned_addr learned my addr 172.21.15.90:0/1512111104 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a37fe700 1 -- 172.21.15.90:0/1512111104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2094009550 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a37fe700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f2094003e60 tx=0x7f2094011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f20940038b0 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20a419ac10 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2094003a10 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f2094019530 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.148+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20a419b770 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.149+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 9) v1 ==== 45554+0+0 (secure 0 0 0) 0x7f20940197f0 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.149+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20a4104790 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.149+0000 7f20a0ff9700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 0x7f208c03abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.150+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f209404c8b0 con 0x7f20a41070b0 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.150+0000 7f20a2ffd700 1 -- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 msgr2=0x7f208c03abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:58:58.802 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.150+0000 7f20a2ffd700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 0x7f208c03abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.155+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f209402b340 con 0x7f20a41070b0 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.351+0000 7f20a2ffd700 1 -- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 msgr2=0x7f208c03abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.351+0000 7f20a2ffd700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 0x7f208c03abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.408+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f20a4075420 con 0x7f20a41070b0 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.408+0000 7f20a0ff9700 1 -- 172.21.15.90:0/1512111104 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) v1 ==== 56+0+88 (secure 0 0 0) 0x7f209404c220 con 0x7f20a41070b0 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 msgr2=0x7f208c03abc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 0x7f208c03abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 msgr2=0x7f20a419a4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f2094003e60 tx=0x7f2094011a30 comp rx=0 tx=0).stop 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 shutdown_connections 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7f208c038700 0x7f208c03abc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 --2- 172.21.15.90:0/1512111104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20a41070b0 0x7f20a419a4d0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 >> 172.21.15.90:0/1512111104 conn(0x7f20a4100aa0 msgr2=0x7f20a41016f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 shutdown_connections 2024-04-06T13:58:58.803 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:58.411+0000 7f20a9d80700 1 -- 172.21.15.90:0/1512111104 wait complete. 2024-04-06T13:58:58.804 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for the mgr to restart... 2024-04-06T13:58:58.804 INFO:teuthology.orchestra.run.smithi090.stdout:Waiting for mgr epoch 9... 2024-04-06T13:58:58.900 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:58 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:58.683+0000 7f6ac27fd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:58:59.168 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:58 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:58.953+0000 7f6ac27fd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T13:58:59.169 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.055+0000 7f6ac27fd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T13:58:59.452 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1512111104' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-06T13:58:59.453 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.168+0000 7f6ac27fd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T13:58:59.453 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.326+0000 7f6ac27fd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T13:58:59.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.452+0000 7f6ac27fd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T13:58:59.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.538+0000 7f6ac27fd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T13:59:00.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:58:59 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:58:59.905+0000 7f6ac27fd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T13:59:00.578 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:00.290+0000 7f6ac27fd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T13:59:00.578 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:00.483+0000 7f6ac27fd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T13:59:01.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:00.575+0000 7f6ac27fd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T13:59:01.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:01 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:01.345+0000 7f6ac27fd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T13:59:01.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:01 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:01.442+0000 7f6ac27fd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T13:59:01.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:01 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:01.593+0000 7f6ac27fd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T13:59:02.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:02 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:02.213+0000 7f6ac27fd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T13:59:02.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:02 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:02.396+0000 7f6ac27fd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T13:59:03.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:02 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:02.590+0000 7f6ac27fd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T13:59:03.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:03 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:03.514+0000 7f6ac27fd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T13:59:04.087 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:03 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:03.805+0000 7f6ac27fd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:59:04.087 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:03 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:03.894+0000 7f6ac27fd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T13:59:04.538 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T13:59:04.087+0000 7f6ac27fd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: Active manager daemon x restarted 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: Activating manager daemon x 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: osdmap e3: 0 total, 0 up, 0 in 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: mgrmap e10: x(active, starting, since 0.0033285s) 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T13:59:04.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: Manager daemon x is now available 2024-04-06T13:59:04.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:05.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T13:59:05.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T13:59:05.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:05 smithi090 ceph-mon[43420]: mgrmap e11: x(active, since 1.00665s) 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout { 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout } 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.145+0000 7fe8b4246700 1 Processor -- start 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b4246700 1 -- start start 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b4246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:05.959 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b4246700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8ac079e80 con 0x7fe8ac1070b0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b1fe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b1fe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54562/0 (socket says 172.21.15.90:54562) 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b1fe2700 1 -- 172.21.15.90:0/3570525182 learned_addr learned my addr 172.21.15.90:0/3570525182 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b1fe2700 1 -- 172.21.15.90:0/3570525182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8ac1099e0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.146+0000 7fe8b1fe2700 1 --2- 172.21.15.90:0/3570525182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fe8a80098a0 tx=0x7fe8a8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b74b87ec2265d76c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.147+0000 7fe8b0fe0700 1 -- 172.21.15.90:0/3570525182 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe8a8004160 con 0x7fe8ac1070b0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.147+0000 7fe8b0fe0700 1 -- 172.21.15.90:0/3570525182 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8a80042c0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.147+0000 7fe8b0fe0700 1 -- 172.21.15.90:0/3570525182 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe8a80119b0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.147+0000 7fe8b4246700 1 -- 172.21.15.90:0/3570525182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 msgr2=0x7fe8ac1094a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.147+0000 7fe8b4246700 1 --2- 172.21.15.90:0/3570525182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7fe8a80098a0 tx=0x7fe8a8009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 -- 172.21.15.90:0/3570525182 shutdown_connections 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 --2- 172.21.15.90:0/3570525182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac1094a0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.960 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 -- 172.21.15.90:0/3570525182 >> 172.21.15.90:0/3570525182 conn(0x7fe8ac100aa0 msgr2=0x7fe8ac102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 -- 172.21.15.90:0/3570525182 shutdown_connections 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 -- 172.21.15.90:0/3570525182 wait complete. 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 Processor -- start 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.148+0000 7fe8b4246700 1 -- start start 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b4246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b4246700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8ac19aa10 con 0x7fe8ac1070b0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b1fe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b1fe2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54570/0 (socket says 172.21.15.90:54570) 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b1fe2700 1 -- 172.21.15.90:0/464951311 learned_addr learned my addr 172.21.15.90:0/464951311 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b1fe2700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8a8009550 con 0x7fe8ac1070b0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b1fe2700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fe8a8003e60 tx=0x7fe8a8011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe8a80038b0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8a8003a10 con 0x7fe8ac1070b0 2024-04-06T13:59:05.961 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8ac19ac10 con 0x7fe8ac1070b0 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fe8a8019530 con 0x7fe8ac1070b0 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.149+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8ac19b770 con 0x7fe8ac1070b0 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.150+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 9) v1 ==== 45554+0+0 (secure 0 0 0) 0x7fe8a80197f0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.151+0000 7fe8a2ffd700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.151+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fe8ac104790 con 0x7fe898038700 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.151+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fe8a804c9f0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.151+0000 7fe8b17e1700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.151+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.352+0000 7fe8b17e1700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.352+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.753+0000 7fe8b17e1700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:58:59.753+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-06T13:59:05.962 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:00.554+0000 7fe8b17e1700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:00.554+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:02.155+0000 7fe8b17e1700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/2246618981 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:02.155+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:04.101+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mgrmap(e 10) v1 ==== 45321+0+0 (secure 0 0 0) 0x7fe8a8019d20 con 0x7fe8ac1070b0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:04.102+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 msgr2=0x7fe89803abc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:04.102+0000 7fe8a2ffd700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.103+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 11) v1 ==== 45448+0+0 (secure 0 0 0) 0x7fe8a802ecb0 con 0x7fe8ac1070b0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.104+0000 7fe8a2ffd700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 0x7fe89803e3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.104+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fe8ac104790 con 0x7fe89803bfc0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.104+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 0x7fe89803e3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.105+0000 7fe8b17e1700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 0x7fe89803e3b0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fe89c003a10 tx=0x7fe89c0092b0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.105+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6910 (secure 0 0 0) 0x7fe8ac104790 con 0x7fe89803bfc0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.112+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7fe8ac19b0b0 con 0x7fe89803bfc0 2024-04-06T13:59:05.963 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8a2ffd700 1 -- 172.21.15.90:0/464951311 <== mgr.14152 v2:172.21.15.90:6800/1992032630 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+52 (secure 0 0 0) 0x7fe8ac19b0b0 con 0x7fe89803bfc0 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 msgr2=0x7fe89803e3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 0x7fe89803e3b0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fe89c003a10 tx=0x7fe89c0092b0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 msgr2=0x7fe8ac19a4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fe8a8003e60 tx=0x7fe8a8011a30 comp rx=0 tx=0).stop 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 shutdown_connections 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe89803bfc0 0x7fe89803e3b0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8ac1070b0 0x7fe8ac19a4d0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 --2- 172.21.15.90:0/464951311 >> [v2:172.21.15.90:6800/2246618981,v1:172.21.15.90:6801/2246618981] conn(0x7fe898038700 0x7fe89803abc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 >> 172.21.15.90:0/464951311 conn(0x7fe8ac100aa0 msgr2=0x7fe8ac1016f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 shutdown_connections 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:05.113+0000 7fe8b4246700 1 -- 172.21.15.90:0/464951311 wait complete. 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:mgr epoch 9 is available 2024-04-06T13:59:05.964 INFO:teuthology.orchestra.run.smithi090.stdout:Generating a dashboard self-signed certificate... 2024-04-06T13:59:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: [06/Apr/2024:13:59:04] ENGINE Bus STARTING 2024-04-06T13:59:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: [06/Apr/2024:13:59:04] ENGINE Serving on https://172.21.15.90:7150 2024-04-06T13:59:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: [06/Apr/2024:13:59:04] ENGINE Serving on http://172.21.15.90:8765 2024-04-06T13:59:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: [06/Apr/2024:13:59:04] ENGINE Bus STARTED 2024-04-06T13:59:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-06T13:59:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-06T13:59:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:06 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:06.983 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f9812982700 1 Processor -- start 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f9812982700 1 -- start start 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f9812982700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f9812982700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f980c1074e0 con 0x7f980c106af0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f980bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f980bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58256/0 (socket says 172.21.15.90:58256) 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.331+0000 7f980bfff700 1 -- 172.21.15.90:0/1525766099 learned_addr learned my addr 172.21.15.90:0/1525766099 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.332+0000 7f980bfff700 1 -- 172.21.15.90:0/1525766099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f980c107620 con 0x7f980c106af0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.332+0000 7f980bfff700 1 --2- 172.21.15.90:0/1525766099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f97fc0098a0 tx=0x7f97fc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=858d934d8d4f9170 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.332+0000 7f980affd700 1 -- 172.21.15.90:0/1525766099 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f97fc004160 con 0x7f980c106af0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.332+0000 7f980affd700 1 -- 172.21.15.90:0/1525766099 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f97fc0042c0 con 0x7f980c106af0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.332+0000 7f980affd700 1 -- 172.21.15.90:0/1525766099 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f97fc0119b0 con 0x7f980c106af0 2024-04-06T13:59:06.984 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- 172.21.15.90:0/1525766099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 msgr2=0x7f980c106f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 --2- 172.21.15.90:0/1525766099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f97fc0098a0 tx=0x7f97fc009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- 172.21.15.90:0/1525766099 shutdown_connections 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 --2- 172.21.15.90:0/1525766099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c106f10 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- 172.21.15.90:0/1525766099 >> 172.21.15.90:0/1525766099 conn(0x7f980c100200 msgr2=0x7f980c102680 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- 172.21.15.90:0/1525766099 shutdown_connections 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- 172.21.15.90:0/1525766099 wait complete. 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 Processor -- start 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.333+0000 7f9812982700 1 -- start start 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f9812982700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f9812982700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f980c19cc50 con 0x7f980c106af0 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f980bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f980bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58260/0 (socket says 172.21.15.90:58260) 2024-04-06T13:59:06.985 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f980bfff700 1 -- 172.21.15.90:0/538330041 learned_addr learned my addr 172.21.15.90:0/538330041 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f980bfff700 1 -- 172.21.15.90:0/538330041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97fc009550 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f980bfff700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f97fc011670 tx=0x7f97fc011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.334+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f97fc003b20 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.335+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f980c19ce50 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.335+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f97fc003c80 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.335+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f97fc019770 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.335+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f980c19d960 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.335+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 11) v1 ==== 45448+0+0 (secure 0 0 0) 0x7f97fc019a30 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.336+0000 7f98097fa700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 0x7f97f403aae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.336+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f97fc04c8c0 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.336+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f980c103f30 con 0x7f980c106af0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.336+0000 7f980b7fe700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 0x7f97f403aae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:06.986 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.336+0000 7f980b7fe700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 0x7f97f403aae0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f9800006fd0 tx=0x7f9800006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.341+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f97fc018db0 con 0x7f980c106af0 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.549+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) v1 -- 0x7f980c19dba0 con 0x7f97f4038620 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.668+0000 7f98097fa700 1 -- 172.21.15.90:0/538330041 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+32 (secure 0 0 0) 0x7f980c19dba0 con 0x7f97f4038620 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 msgr2=0x7f97f403aae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 0x7f97f403aae0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f9800006fd0 tx=0x7f9800006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 msgr2=0x7f980c19c710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f97fc011670 tx=0x7f97fc011a30 comp rx=0 tx=0).stop 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 shutdown_connections 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f97f4038620 0x7f97f403aae0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 --2- 172.21.15.90:0/538330041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f980c106af0 0x7f980c19c710 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 >> 172.21.15.90:0/538330041 conn(0x7f980c100200 msgr2=0x7f980c192e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:06.987 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 shutdown_connections 2024-04-06T13:59:06.988 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:06.671+0000 7f9812982700 1 -- 172.21.15.90:0/538330041 wait complete. 2024-04-06T13:59:06.988 INFO:teuthology.orchestra.run.smithi090.stdout:Creating initial admin user... 2024-04-06T13:59:07.992 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:07 smithi090 ceph-mon[43420]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:07.992 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:07.992 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:07.992 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:07.992 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:07 smithi090 ceph-mon[43420]: mgrmap e12: x(active, since 2s) 2024-04-06T13:59:08.190 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$2w3AbD6ue7Mebl5fbiuWz.OgfwlTjZEi9kzr2ybO5d406P3tC4vp2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712411947, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.356+0000 7f1060414700 1 Processor -- start 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.356+0000 7f1060414700 1 -- start start 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f1060414700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f1060414700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1058105ef0 con 0x7f1058105500 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105e1b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105e1b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58272/0 (socket says 172.21.15.90:58272) 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105e1b0700 1 -- 172.21.15.90:0/1687245213 learned_addr learned my addr 172.21.15.90:0/1687245213 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105e1b0700 1 -- 172.21.15.90:0/1687245213 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1058106700 con 0x7f1058105500 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105e1b0700 1 --2- 172.21.15.90:0/1687245213 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f10500098a0 tx=0x7f1050009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d8c8e433f6dc01e3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105d1ae700 1 -- 172.21.15.90:0/1687245213 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1050004160 con 0x7f1058105500 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105d1ae700 1 -- 172.21.15.90:0/1687245213 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f10500042c0 con 0x7f1058105500 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.357+0000 7f105d1ae700 1 -- 172.21.15.90:0/1687245213 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f10500119b0 con 0x7f1058105500 2024-04-06T13:59:08.191 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 -- 172.21.15.90:0/1687245213 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 msgr2=0x7f1058105920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 --2- 172.21.15.90:0/1687245213 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f10500098a0 tx=0x7f1050009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 -- 172.21.15.90:0/1687245213 shutdown_connections 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 --2- 172.21.15.90:0/1687245213 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f1058105920 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 -- 172.21.15.90:0/1687245213 >> 172.21.15.90:0/1687245213 conn(0x7f1058100aa0 msgr2=0x7f1058102ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 -- 172.21.15.90:0/1687245213 shutdown_connections 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.358+0000 7f1060414700 1 -- 172.21.15.90:0/1687245213 wait complete. 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f1060414700 1 Processor -- start 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f1060414700 1 -- start start 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f1060414700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:08.192 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f1060414700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f105819aa50 con 0x7f1058105500 2024-04-06T13:59:08.193 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f105e1b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:08.193 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f105e1b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58278/0 (socket says 172.21.15.90:58278) 2024-04-06T13:59:08.193 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f105e1b0700 1 -- 172.21.15.90:0/2748095798 learned_addr learned my addr 172.21.15.90:0/2748095798 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:08.193 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f105e1b0700 1 -- 172.21.15.90:0/2748095798 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1050009550 con 0x7f1058105500 2024-04-06T13:59:08.193 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.359+0000 7f105e1b0700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f1050003e60 tx=0x7f1050011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f10500038b0 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1050003a10 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f105819ac50 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f1050019530 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f105819b7b0 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.360+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 12) v1 ==== 45554+0+0 (secure 0 0 0) 0x7f105001a980 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.361+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1058104790 con 0x7f1058105500 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.361+0000 7f10577fe700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 0x7f104003abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:08.194 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.361+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f105004c9f0 con 0x7f1058105500 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.361+0000 7f105d9af700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 0x7f104003abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.361+0000 7f105d9af700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 0x7f104003abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f104c006fd0 tx=0x7f104c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.365+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f1050003b80 con 0x7f1058105500 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.586+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f1058062200 con 0x7f1040038700 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.814+0000 7f10577fe700 1 -- 172.21.15.90:0/2748095798 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+252 (secure 0 0 0) 0x7f1058062200 con 0x7f1040038700 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 msgr2=0x7f104003abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:08.195 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 0x7f104003abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f104c006fd0 tx=0x7f104c006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 msgr2=0x7f105819a510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f1050003e60 tx=0x7f1050011a30 comp rx=0 tx=0).stop 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 shutdown_connections 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1040038700 0x7f104003abc0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 --2- 172.21.15.90:0/2748095798 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1058105500 0x7f105819a510 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 >> 172.21.15.90:0/2748095798 conn(0x7f1058100aa0 msgr2=0x7f1058101a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 shutdown_connections 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:07.816+0000 7f1060414700 1 -- 172.21.15.90:0/2748095798 wait complete. 2024-04-06T13:59:08.196 INFO:teuthology.orchestra.run.smithi090.stdout:Fetching dashboard port number... 2024-04-06T13:59:09.020 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:08 smithi090 ceph-mon[43420]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:09.020 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:09.020 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:08 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4268484816' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-06T13:59:09.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stdout 8443 2024-04-06T13:59:09.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa67475a700 1 Processor -- start 2024-04-06T13:59:09.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa67475a700 1 -- start start 2024-04-06T13:59:09.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa67475a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:09.056 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa67475a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66c1052c0 con 0x7fa66c1048d0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa6724f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa6724f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58284/0 (socket says 172.21.15.90:58284) 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.508+0000 7fa6724f6700 1 -- 172.21.15.90:0/1044845603 learned_addr learned my addr 172.21.15.90:0/1044845603 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa6724f6700 1 -- 172.21.15.90:0/1044845603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa66c105400 con 0x7fa66c1048d0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa6724f6700 1 --2- 172.21.15.90:0/1044845603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa6680098a0 tx=0x7fa668009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f21059c356acc772 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa6714f4700 1 -- 172.21.15.90:0/1044845603 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa668004160 con 0x7fa66c1048d0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa6714f4700 1 -- 172.21.15.90:0/1044845603 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa6680042c0 con 0x7fa66c1048d0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa6714f4700 1 -- 172.21.15.90:0/1044845603 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa6680119b0 con 0x7fa66c1048d0 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa67475a700 1 -- 172.21.15.90:0/1044845603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 msgr2=0x7fa66c104cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa67475a700 1 --2- 172.21.15.90:0/1044845603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa6680098a0 tx=0x7fa668009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa67475a700 1 -- 172.21.15.90:0/1044845603 shutdown_connections 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa67475a700 1 --2- 172.21.15.90:0/1044845603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c104cf0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.057 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.509+0000 7fa67475a700 1 -- 172.21.15.90:0/1044845603 >> 172.21.15.90:0/1044845603 conn(0x7fa66c1002b0 msgr2=0x7fa66c1026d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 -- 172.21.15.90:0/1044845603 shutdown_connections 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 -- 172.21.15.90:0/1044845603 wait complete. 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 Processor -- start 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 -- start start 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa67475a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66c19a980 con 0x7fa66c1048d0 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa6724f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa6724f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58300/0 (socket says 172.21.15.90:58300) 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.510+0000 7fa6724f6700 1 -- 172.21.15.90:0/4268484816 learned_addr learned my addr 172.21.15.90:0/4268484816 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa6724f6700 1 -- 172.21.15.90:0/4268484816 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa668009550 con 0x7fa66c1048d0 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa6724f6700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa668006af0 tx=0x7fa668011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa668003a40 con 0x7fa66c1048d0 2024-04-06T13:59:09.058 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa668003ba0 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa668019690 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa66c19abe0 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.511+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa66c19b6e0 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 12) v1 ==== 45554+0+0 (secure 0 0 0) 0x7fa66801aa60 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa6637fe700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 0x7fa65803ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa66c10ab80 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fa66804ca60 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa671cf5700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 0x7fa65803ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.512+0000 7fa671cf5700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 0x7fa65803ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fa65c006fd0 tx=0x7fa65c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.517+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fa66802d430 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.708+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) v1 -- 0x7fa66c10ab20 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.709+0000 7fa6637fe700 1 -- 172.21.15.90:0/4268484816 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) v1 ==== 112+0+5 (secure 0 0 0) 0x7fa668023390 con 0x7fa66c1048d0 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 msgr2=0x7fa65803ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:09.059 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 0x7fa65803ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fa65c006fd0 tx=0x7fa65c006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 msgr2=0x7fa66c19a440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fa668006af0 tx=0x7fa668011a10 comp rx=0 tx=0).stop 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 shutdown_connections 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa658038750 0x7fa65803ac10 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 --2- 172.21.15.90:0/4268484816 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1048d0 0x7fa66c19a440 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 >> 172.21.15.90:0/4268484816 conn(0x7fa66c1002b0 msgr2=0x7fa66c100e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 shutdown_connections 2024-04-06T13:59:09.060 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:08.711+0000 7fa67475a700 1 -- 172.21.15.90:0/4268484816 wait complete. 2024-04-06T13:59:09.063 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-06T13:59:09.063 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout disabled 2024-04-06T13:59:09.070 INFO:teuthology.orchestra.run.smithi090.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-06T13:59:09.070 INFO:teuthology.orchestra.run.smithi090.stdout:systemctl: stdout inactive 2024-04-06T13:59:09.070 INFO:teuthology.orchestra.run.smithi090.stdout:firewalld.service is not enabled 2024-04-06T13:59:09.070 INFO:teuthology.orchestra.run.smithi090.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout:Ceph Dashboard is now available at: 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout: URL: https://smithi090:8443/ 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout: User: admin 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout: Password: 66zc9vm959 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:09.072 INFO:teuthology.orchestra.run.smithi090.stdout:Saving cluster configuration to /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config directory 2024-04-06T13:59:09.073 INFO:teuthology.orchestra.run.smithi090.stdout:Enabling autotune for osd_memory_target 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f706d15a700 1 Processor -- start 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f706d15a700 1 -- start start 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f706d15a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f706d15a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f706807a810 con 0x7f706807be20 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f7066d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f7066d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58310/0 (socket says 172.21.15.90:58310) 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f7066d9d700 1 -- 172.21.15.90:0/3664058335 learned_addr learned my addr 172.21.15.90:0/3664058335 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.435+0000 7f7066d9d700 1 -- 172.21.15.90:0/3664058335 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f706807a950 con 0x7f706807be20 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f7066d9d700 1 --2- 172.21.15.90:0/3664058335 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f70580098a0 tx=0x7f7058009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52e733ee46696a14 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f7065d9b700 1 -- 172.21.15.90:0/3664058335 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7058004160 con 0x7f706807be20 2024-04-06T13:59:09.998 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f7065d9b700 1 -- 172.21.15.90:0/3664058335 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f70580042c0 con 0x7f706807be20 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f7065d9b700 1 -- 172.21.15.90:0/3664058335 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f70580119b0 con 0x7f706807be20 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 -- 172.21.15.90:0/3664058335 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 msgr2=0x7f706807a2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 --2- 172.21.15.90:0/3664058335 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f70580098a0 tx=0x7f7058009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 -- 172.21.15.90:0/3664058335 shutdown_connections 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 --2- 172.21.15.90:0/3664058335 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706807a2d0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 -- 172.21.15.90:0/3664058335 >> 172.21.15.90:0/3664058335 conn(0x7f7068101b40 msgr2=0x7f7068103f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 -- 172.21.15.90:0/3664058335 shutdown_connections 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.436+0000 7f706d15a700 1 -- 172.21.15.90:0/3664058335 wait complete. 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.437+0000 7f706d15a700 1 Processor -- start 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.437+0000 7f706d15a700 1 -- start start 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.437+0000 7f706d15a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f706d15a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f706819cb80 con 0x7f706807be20 2024-04-06T13:59:09.999 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f7066d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f7066d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58326/0 (socket says 172.21.15.90:58326) 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f7066d9d700 1 -- 172.21.15.90:0/2417232939 learned_addr learned my addr 172.21.15.90:0/2417232939 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f7066d9d700 1 -- 172.21.15.90:0/2417232939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7058009550 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f7066d9d700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f7058011670 tx=0x7f7058011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7058003b20 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7058003c80 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f706819cd80 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7058019770 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.438+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f706819d8e0 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.439+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70680623e0 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.439+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 12) v1 ==== 45554+0+0 (secure 0 0 0) 0x7f7058019a30 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.439+0000 7f704ffff700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 0x7f705003ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.439+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f705804cbf0 con 0x7f706807be20 2024-04-06T13:59:10.000 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.440+0000 7f706659c700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 0x7f705003ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.440+0000 7f706659c700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 0x7f705003ac10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f705c006fd0 tx=0x7f705c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.444+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f70580189a0 con 0x7f706807be20 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.635+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) v1 -- 0x7f706810cbf0 con 0x7f706807be20 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.635+0000 7f704ffff700 1 -- 172.21.15.90:0/2417232939 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) v1 ==== 127+0+0 (secure 0 0 0) 0x7f7058018350 con 0x7f706807be20 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 msgr2=0x7f705003ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 0x7f705003ac10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f705c006fd0 tx=0x7f705c006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 msgr2=0x7f706819c640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f7058011670 tx=0x7f7058011a30 comp rx=0 tx=0).stop 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 shutdown_connections 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7050038750 0x7f705003ac10 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 --2- 172.21.15.90:0/2417232939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f706807be20 0x7f706819c640 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 >> 172.21.15.90:0/2417232939 conn(0x7f7068101b40 msgr2=0x7f7068102740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:10.001 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 shutdown_connections 2024-04-06T13:59:10.002 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:09.637+0000 7f706d15a700 1 -- 172.21.15.90:0/2417232939 wait complete. 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe5141e700 1 Processor -- start 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe5141e700 1 -- start start 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe5141e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe5141e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe4c107040 con 0x7efe4c106650 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58332/0 (socket says 172.21.15.90:58332) 2024-04-06T13:59:11.007 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.334+0000 7efe4affd700 1 -- 172.21.15.90:0/1164071127 learned_addr learned my addr 172.21.15.90:0/1164071127 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe4affd700 1 -- 172.21.15.90:0/1164071127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe4c107800 con 0x7efe4c106650 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe4affd700 1 --2- 172.21.15.90:0/1164071127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7efe400098a0 tx=0x7efe40009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c531dd7184be7212 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe49ffb700 1 -- 172.21.15.90:0/1164071127 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe40004160 con 0x7efe4c106650 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe49ffb700 1 -- 172.21.15.90:0/1164071127 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe400042c0 con 0x7efe4c106650 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe49ffb700 1 -- 172.21.15.90:0/1164071127 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe400119b0 con 0x7efe4c106650 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 -- 172.21.15.90:0/1164071127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 msgr2=0x7efe4c106a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 --2- 172.21.15.90:0/1164071127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7efe400098a0 tx=0x7efe40009bb0 comp rx=0 tx=0).stop 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 -- 172.21.15.90:0/1164071127 shutdown_connections 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 --2- 172.21.15.90:0/1164071127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c106a70 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 -- 172.21.15.90:0/1164071127 >> 172.21.15.90:0/1164071127 conn(0x7efe4c101c90 msgr2=0x7efe4c1040b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.335+0000 7efe5141e700 1 -- 172.21.15.90:0/1164071127 shutdown_connections 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe5141e700 1 -- 172.21.15.90:0/1164071127 wait complete. 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe5141e700 1 Processor -- start 2024-04-06T13:59:11.008 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe5141e700 1 -- start start 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe5141e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe5141e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe4c19a980 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.336+0000 7efe4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58344/0 (socket says 172.21.15.90:58344) 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe4affd700 1 -- 172.21.15.90:0/3523356257 learned_addr learned my addr 172.21.15.90:0/3523356257 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe4affd700 1 -- 172.21.15.90:0/3523356257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe40009550 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe4affd700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7efe40006af0 tx=0x7efe40011a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe40003a40 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe40003ba0 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe4c19ab80 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe40019690 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.337+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe4c19b6e0 con 0x7efe4c106650 2024-04-06T13:59:11.009 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7efe40019950 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe33fff700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 0x7efe3403ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efe4c1058e0 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7efe4004ce40 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe4a7fc700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 0x7efe3403ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.338+0000 7efe4a7fc700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 0x7efe3403ac60 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7efe3c006fd0 tx=0x7efe3c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.343+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efe40018760 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.643+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) v1 -- 0x7efe4c10ac70 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.645+0000 7efe33fff700 1 -- 172.21.15.90:0/3523356257 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v28)=0 set mgr/dashboard/cluster/status v28) v1 ==== 153+0+0 (secure 0 0 0) 0x7efe40016cd0 con 0x7efe4c106650 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 msgr2=0x7efe3403ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 0x7efe3403ac60 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7efe3c006fd0 tx=0x7efe3c006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 msgr2=0x7efe4c19a440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7efe40006af0 tx=0x7efe40011a10 comp rx=0 tx=0).stop 2024-04-06T13:59:11.010 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 shutdown_connections 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe340387a0 0x7efe3403ac60 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 --2- 172.21.15.90:0/3523356257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe4c106650 0x7efe4c19a440 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.647+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 >> 172.21.15.90:0/3523356257 conn(0x7efe4c101c90 msgr2=0x7efe4c102820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.648+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 shutdown_connections 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr 2024-04-06T13:59:10.648+0000 7efe5141e700 1 -- 172.21.15.90:0/3523356257 wait complete. 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:Or, if you are only running a single cluster on this host: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.011 INFO:teuthology.orchestra.run.smithi090.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: ceph telemetry on 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout:For more information see: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:11.012 INFO:teuthology.orchestra.run.smithi090.stdout:Bootstrap complete. 2024-04-06T13:59:11.039 INFO:tasks.cephadm:Fetching config... 2024-04-06T13:59:11.040 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:11.040 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-06T13:59:11.059 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-06T13:59:11.059 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:11.060 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-06T13:59:11.116 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-06T13:59:11.117 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:11.117 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-06T13:59:11.186 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-06T13:59:11.186 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:11.186 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-06T13:59:11.243 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:11 smithi090 ceph-mon[43420]: mgrmap e13: x(active, since 6s) 2024-04-06T13:59:11.244 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3523356257' entity='client.admin' 2024-04-06T13:59:11.245 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-06T13:59:11.245 DEBUG:teuthology.orchestra.run.smithi090:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-06T13:59:11.325 INFO:teuthology.orchestra.run.smithi090.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:11.337 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-06T13:59:11.382 INFO:teuthology.orchestra.run.smithi105.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:11.395 DEBUG:teuthology.orchestra.run.smithi134:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-06T13:59:11.443 INFO:teuthology.orchestra.run.smithi134.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkkcpC/KDirPRLmUKo5AiyYeMgcNgIvYqYPVDnScS1AT3NreaQEitNHJxpwQUldRnjsJWiIS5mE7OexvqrTtI5a9CxWKoc7AXiHI9xOScdsrnXcLUZaVqqGVOxd9QGa7P5ywH0ikP/QSLfYno95CiWsnyR1tjWSd/O61chcR7qVb1oUkgIIjTn0Wlwo2ZbyonxTEEh6itKqhtB2JwZMGR3P2VHHV1jSKO2qAhrEXnIbSN9Lte082h3Ove5ISI7SII0SzWG08MuE7Zx/brAWQ+D4k02KUwCp9vYpCsY1irOs7NmWeHePIAlFG+Q13ZfhqKFpprJBKS03h1/HoChsMKg7X3opP7OWmPNYtmZeWP143RdS1+Nvuoy0ms0hDa/UEoVYpkIfhzZlNlHBFzLc+cgNh0U351ycI6EwiCEIpDa2VP9mVQBU3fjUC2lVbeA9MQQz7BBTiajNwVWPYcOOR80n2gXWSKSnnelheEB+DRpirIkVVmZztss06gih7NjvrE= ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:11.452 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-06T13:59:11.675 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:12.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 -- 172.21.15.90:0/1748516265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 msgr2=0x7fcef8100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:12.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/1748516265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8100b60 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fcee0009910 tx=0x7fcee0009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:12.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 -- 172.21.15.90:0/1748516265 shutdown_connections 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/1748516265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8100b60 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 -- 172.21.15.90:0/1748516265 >> 172.21.15.90:0/1748516265 conn(0x7fcef80fbce0 msgr2=0x7fcef80fe120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 -- 172.21.15.90:0/1748516265 shutdown_connections 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.952+0000 7fcefd1f4700 1 -- 172.21.15.90:0/1748516265 wait complete. 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcefd1f4700 1 Processor -- start 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcefd1f4700 1 -- start start 2024-04-06T13:59:12.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcefd1f4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcefd1f4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcef8198330 con 0x7fcef8100740 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcef6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcef6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58372/0 (socket says 172.21.15.90:58372) 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcef6d9d700 1 -- 172.21.15.90:0/3150192211 learned_addr learned my addr 172.21.15.90:0/3150192211 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.953+0000 7fcef6d9d700 1 -- 172.21.15.90:0/3150192211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcee00095f0 con 0x7fcef8100740 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fcef6d9d700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fcee0005950 tx=0x7fcee00057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fcee001d070 con 0x7fcef8100740 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcef8198530 con 0x7fcef8100740 2024-04-06T13:59:12.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcee000bc10 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fcee0003a10 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.954+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcef81989d0 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.955+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcef8191860 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.955+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7fcee0004390 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.955+0000 7fceeffff700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 0x7fcee403ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.955+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fcee004c920 con 0x7fcef8100740 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.955+0000 7fcef659c700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 0x7fcee403ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:12.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.956+0000 7fcef659c700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 0x7fcee403ac60 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fcee8006fd0 tx=0x7fcee8006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:12.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:12.960+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fcee000ba70 con 0x7fcef8100740 2024-04-06T13:59:13.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.166+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) v1 -- 0x7fcef8106740 con 0x7fcef8100740 2024-04-06T13:59:13.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.170+0000 7fceeffff700 1 -- 172.21.15.90:0/3150192211 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) v1 ==== 125+0+0 (secure 0 0 0) 0x7fcee0026030 con 0x7fcef8100740 2024-04-06T13:59:13.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 msgr2=0x7fcee403ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:13.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 0x7fcee403ac60 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fcee8006fd0 tx=0x7fcee8006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:13.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 msgr2=0x7fcef8197df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fcee0005950 tx=0x7fcee00057c0 comp rx=0 tx=0).stop 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 shutdown_connections 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcee40387a0 0x7fcee403ac60 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 --2- 172.21.15.90:0/3150192211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcef8100740 0x7fcef8197df0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 >> 172.21.15.90:0/3150192211 conn(0x7fcef80fbce0 msgr2=0x7fcef80fe120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 shutdown_connections 2024-04-06T13:59:13.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:13.174+0000 7fcefd1f4700 1 -- 172.21.15.90:0/3150192211 wait complete. 2024-04-06T13:59:13.539 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-06T13:59:13.539 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-06T13:59:13.765 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:14.454 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:14 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3150192211' entity='client.admin' 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 -- 172.21.15.90:0/3475242574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 msgr2=0x7f6f80100b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 --2- 172.21.15.90:0/3475242574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80100b60 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f6f70009910 tx=0x7f6f70009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 -- 172.21.15.90:0/3475242574 shutdown_connections 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 --2- 172.21.15.90:0/3475242574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80100b60 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 -- 172.21.15.90:0/3475242574 >> 172.21.15.90:0/3475242574 conn(0x7f6f800fbce0 msgr2=0x7f6f800fe120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 -- 172.21.15.90:0/3475242574 shutdown_connections 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.038+0000 7f6f8785e700 1 -- 172.21.15.90:0/3475242574 wait complete. 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f8785e700 1 Processor -- start 2024-04-06T13:59:15.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f8785e700 1 -- start start 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f8785e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f8785e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f80074f60 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f855fa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f855fa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40080/0 (socket says 172.21.15.90:40080) 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f855fa700 1 -- 172.21.15.90:0/1741062310 learned_addr learned my addr 172.21.15.90:0/1741062310 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f855fa700 1 -- 172.21.15.90:0/1741062310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f700095f0 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.039+0000 7f6f855fa700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6f70004f70 tx=0x7f6f70004fa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f6f7001d070 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6f7000be20 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f800731a0 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f6f70022df0 con 0x7f6f80100740 2024-04-06T13:59:15.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f80073640 con 0x7f6f80100740 2024-04-06T13:59:15.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.040+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f6f7000f460 con 0x7f6f80100740 2024-04-06T13:59:15.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.041+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f80106d90 con 0x7f6f80100740 2024-04-06T13:59:15.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.041+0000 7f6f767fc700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 0x7f6f6c03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:15.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.041+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f6f7004cbb0 con 0x7f6f80100740 2024-04-06T13:59:15.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.041+0000 7f6f84df9700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 0x7f6f6c03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:15.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.041+0000 7f6f84df9700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 0x7f6f6c03acb0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f6f7c006fd0 tx=0x7f6f7c006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:15.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.045+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6f70020960 con 0x7f6f80100740 2024-04-06T13:59:15.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.240+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) v1 -- 0x7f6f80073880 con 0x7f6f6c0387f0 2024-04-06T13:59:15.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.243+0000 7f6f767fc700 1 -- 172.21.15.90:0/1741062310 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f6f80073880 con 0x7f6f6c0387f0 2024-04-06T13:59:15.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 msgr2=0x7f6f6c03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 0x7f6f6c03acb0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f6f7c006fd0 tx=0x7f6f7c006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 msgr2=0x7f6f80074a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6f70004f70 tx=0x7f6f70004fa0 comp rx=0 tx=0).stop 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 shutdown_connections 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6f6c0387f0 0x7f6f6c03acb0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 --2- 172.21.15.90:0/1741062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6f80100740 0x7f6f80074a20 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 >> 172.21.15.90:0/1741062310 conn(0x7f6f800fbce0 msgr2=0x7f6f800fc880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.245+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 shutdown_connections 2024-04-06T13:59:15.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:15.246+0000 7f6f8785e700 1 -- 172.21.15.90:0/1741062310 wait complete. 2024-04-06T13:59:15.641 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi105 2024-04-06T13:59:15.641 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:59:15.641 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.conf 2024-04-06T13:59:15.662 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:59:15.662 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:15.722 INFO:tasks.cephadm:Adding host smithi105 to orchestrator... 2024-04-06T13:59:15.722 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch host add smithi105 2024-04-06T13:59:15.968 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi090", "name": "osd_memory_target"}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:16.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:15 smithi090 ceph-mon[43420]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-06T13:59:17.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 -- 172.21.15.90:0/433671263 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 msgr2=0x7f7be4106ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:17.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 --2- 172.21.15.90:0/433671263 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4106ab0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f7bd8009910 tx=0x7f7bd8009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:17.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 -- 172.21.15.90:0/433671263 shutdown_connections 2024-04-06T13:59:17.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 --2- 172.21.15.90:0/433671263 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4106ab0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:17.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 -- 172.21.15.90:0/433671263 >> 172.21.15.90:0/433671263 conn(0x7f7be4075160 msgr2=0x7f7be4075570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:17.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 -- 172.21.15.90:0/433671263 shutdown_connections 2024-04-06T13:59:17.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.043+0000 7f7be8b87700 1 -- 172.21.15.90:0/433671263 wait complete. 2024-04-06T13:59:17.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.044+0000 7f7be8b87700 1 Processor -- start 2024-04-06T13:59:17.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.044+0000 7f7be8b87700 1 -- start start 2024-04-06T13:59:17.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.044+0000 7f7be8b87700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.044+0000 7f7be8b87700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7be4196480 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.044+0000 7f7be259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40088/0 (socket says 172.21.15.90:40088) 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be259c700 1 -- 172.21.15.90:0/1873056849 learned_addr learned my addr 172.21.15.90:0/1873056849 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be259c700 1 -- 172.21.15.90:0/1873056849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bd80095f0 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be259c700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7bd8006010 tx=0x7f7bd8004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7bd801d070 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7be41966e0 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7bd800bbb0 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7bd8003810 con 0x7f7be41066d0 2024-04-06T13:59:17.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.045+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7be4196b60 con 0x7f7be41066d0 2024-04-06T13:59:17.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.046+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f7bd8003a10 con 0x7f7be41066d0 2024-04-06T13:59:17.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.046+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7be4101e10 con 0x7f7be41066d0 2024-04-06T13:59:17.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.046+0000 7f7bd37fe700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 0x7f7bcc03ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:17.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.046+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f7bd804c430 con 0x7f7be41066d0 2024-04-06T13:59:17.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.046+0000 7f7be1d9b700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 0x7f7bcc03ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:17.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.047+0000 7f7be1d9b700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 0x7f7bcc03ac60 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7bd4006fd0 tx=0x7f7bd4006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:17.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.051+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7bd8029770 con 0x7f7be41066d0 2024-04-06T13:59:17.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:17.248+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi105", "target": ["mon-mgr", ""]}) v1 -- 0x7f7be410a9c0 con 0x7f7bcc0387a0 2024-04-06T13:59:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T13:59:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:18.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:18 smithi090 ceph-mon[43420]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.376+0000 7f7bd37fe700 1 -- 172.21.15.90:0/1873056849 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f7be410a9c0 con 0x7f7bcc0387a0 2024-04-06T13:59:19.377 INFO:teuthology.orchestra.run.smithi090.stdout:Added host 'smithi105' with addr '172.21.15.105' 2024-04-06T13:59:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 msgr2=0x7f7bcc03ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 0x7f7bcc03ac60 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7bd4006fd0 tx=0x7f7bd4006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 msgr2=0x7f7be4195f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7bd8006010 tx=0x7f7bd8004ff0 comp rx=0 tx=0).stop 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 shutdown_connections 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7bcc0387a0 0x7f7bcc03ac60 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 --2- 172.21.15.90:0/1873056849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7be41066d0 0x7f7be4195f40 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 >> 172.21.15.90:0/1873056849 conn(0x7f7be4075160 msgr2=0x7f7be40fcc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 shutdown_connections 2024-04-06T13:59:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:19.378+0000 7f7be8b87700 1 -- 172.21.15.90:0/1873056849 wait complete. 2024-04-06T13:59:19.525 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:19 smithi090 ceph-mon[43420]: Deploying cephadm binary to smithi105 2024-04-06T13:59:19.753 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-06T13:59:19.970 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:20.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:20.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:20 smithi090 ceph-mon[43420]: Added host smithi105 2024-04-06T13:59:20.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:20.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:20.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:21.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/2493467090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 msgr2=0x7fb0e0106aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/2493467090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0106aa0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7fb0d4009910 tx=0x7fb0d4009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/2493467090 shutdown_connections 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/2493467090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0106aa0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/2493467090 >> 172.21.15.90:0/2493467090 conn(0x7fb0e00fc1b0 msgr2=0x7fb0e00fe5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/2493467090 shutdown_connections 2024-04-06T13:59:21.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.216+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/2493467090 wait complete. 2024-04-06T13:59:21.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.217+0000 7fb0e4fc3700 1 Processor -- start 2024-04-06T13:59:21.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.217+0000 7fb0e4fc3700 1 -- start start 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0e4fc3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0e4fc3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0e01964a0 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0de59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0de59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40114/0 (socket says 172.21.15.90:40114) 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0de59c700 1 -- 172.21.15.90:0/1359195359 learned_addr learned my addr 172.21.15.90:0/1359195359 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0de59c700 1 -- 172.21.15.90:0/1359195359 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0d40095f0 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0de59c700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb0d4004f40 tx=0x7fb0d4005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0d401d070 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0e0196700 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb0d400bbb0 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb0d4003810 con 0x7fb0e01066c0 2024-04-06T13:59:21.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.218+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0e0196b80 con 0x7fb0e01066c0 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.219+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7fb0d4003a90 con 0x7fb0e01066c0 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.219+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0e0101e00 con 0x7fb0e01066c0 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.219+0000 7fb0cf7fe700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 0x7fb0c803ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.219+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fb0d404c430 con 0x7fb0e01066c0 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.219+0000 7fb0ddd9b700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 0x7fb0c803ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:21.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.220+0000 7fb0ddd9b700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 0x7fb0c803ac60 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fb0d0006fd0 tx=0x7fb0d0006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:21.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.224+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb0d4029770 con 0x7fb0e01066c0 2024-04-06T13:59:21.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.423+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fb0e010a9c0 con 0x7fb0c80387a0 2024-04-06T13:59:21.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.424+0000 7fb0cf7fe700 1 -- 172.21.15.90:0/1359195359 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+160 (secure 0 0 0) 0x7fb0e010a9c0 con 0x7fb0c80387a0 2024-04-06T13:59:21.425 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:21.425 INFO:teuthology.orchestra.run.smithi090.stdout:[{"addr": "172.21.15.90", "hostname": "smithi090", "labels": [], "status": ""}, {"addr": "172.21.15.105", "hostname": "smithi105", "labels": [], "status": ""}] 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 msgr2=0x7fb0c803ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 0x7fb0c803ac60 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fb0d0006fd0 tx=0x7fb0d0006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 msgr2=0x7fb0e0195f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb0d4004f40 tx=0x7fb0d4005020 comp rx=0 tx=0).stop 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 shutdown_connections 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb0c80387a0 0x7fb0c803ac60 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 --2- 172.21.15.90:0/1359195359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0e01066c0 0x7fb0e0195f60 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:21.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 >> 172.21.15.90:0/1359195359 conn(0x7fb0e00fc1b0 msgr2=0x7fb0e01082f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:21.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 shutdown_connections 2024-04-06T13:59:21.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:21.426+0000 7fb0e4fc3700 1 -- 172.21.15.90:0/1359195359 wait complete. 2024-04-06T13:59:21.821 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi134 2024-04-06T13:59:21.821 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:59:21.821 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.conf 2024-04-06T13:59:21.842 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:59:21.843 DEBUG:teuthology.orchestra.run.smithi134:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:21.905 INFO:tasks.cephadm:Adding host smithi134 to orchestrator... 2024-04-06T13:59:21.905 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch host add smithi134 2024-04-06T13:59:22.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:22.120 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:22.984 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:22 smithi090 ceph-mon[43420]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 -- 172.21.15.90:0/4150985940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 msgr2=0x7fb858105a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 --2- 172.21.15.90:0/4150985940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858105a50 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb848009910 tx=0x7fb848009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 -- 172.21.15.90:0/4150985940 shutdown_connections 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 --2- 172.21.15.90:0/4150985940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858105a50 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 -- 172.21.15.90:0/4150985940 >> 172.21.15.90:0/4150985940 conn(0x7fb8580fb0d0 msgr2=0x7fb8580fd4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 -- 172.21.15.90:0/4150985940 shutdown_connections 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.151+0000 7fb85cc12700 1 -- 172.21.15.90:0/4150985940 wait complete. 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85cc12700 1 Processor -- start 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85cc12700 1 -- start start 2024-04-06T13:59:23.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85cc12700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85cc12700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb858069ac0 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40128/0 (socket says 172.21.15.90:40128) 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85659c700 1 -- 172.21.15.90:0/1140459234 learned_addr learned my addr 172.21.15.90:0/1140459234 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85659c700 1 -- 172.21.15.90:0/1140459234 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8480095f0 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb85659c700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fb848004f40 tx=0x7fb848005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.152+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb84801d070 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.153+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb84800b750 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.153+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb848003810 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.153+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb858197340 con 0x7fb858105670 2024-04-06T13:59:23.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.153+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb858197840 con 0x7fb858105670 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.153+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7fb848003ad0 con 0x7fb858105670 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.154+0000 7fb84f7fe700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 0x7fb84403ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.154+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fb84804c480 con 0x7fb858105670 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.154+0000 7fb855d9b700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 0x7fb84403ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.154+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb858100d20 con 0x7fb858105670 2024-04-06T13:59:23.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.154+0000 7fb855d9b700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 0x7fb84403ac60 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fb840006fd0 tx=0x7fb840006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:23.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.159+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb848026020 con 0x7fb858105670 2024-04-06T13:59:23.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:23.360+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}) v1 -- 0x7fb858069160 con 0x7fb8440387a0 2024-04-06T13:59:24.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: Updating smithi105:/etc/ceph/ceph.conf 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi134", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:24.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:23 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:25 smithi090 ceph-mon[43420]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:25 smithi090 ceph-mon[43420]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T13:59:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:25 smithi090 ceph-mon[43420]: Deploying cephadm binary to smithi134 2024-04-06T13:59:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:25 smithi090 ceph-mon[43420]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.560+0000 7fb84f7fe700 1 -- 172.21.15.90:0/1140459234 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7fb858069160 con 0x7fb8440387a0 2024-04-06T13:59:25.561 INFO:teuthology.orchestra.run.smithi090.stdout:Added host 'smithi134' with addr '172.21.15.134' 2024-04-06T13:59:25.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 msgr2=0x7fb84403ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 0x7fb84403ac60 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fb840006fd0 tx=0x7fb840006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 msgr2=0x7fb858196e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fb848004f40 tx=0x7fb848005020 comp rx=0 tx=0).stop 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 shutdown_connections 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb8440387a0 0x7fb84403ac60 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 --2- 172.21.15.90:0/1140459234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb858105670 0x7fb858196e00 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 >> 172.21.15.90:0/1140459234 conn(0x7fb8580fb0d0 msgr2=0x7fb8580fb4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 shutdown_connections 2024-04-06T13:59:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:25.562+0000 7fb85cc12700 1 -- 172.21.15.90:0/1140459234 wait complete. 2024-04-06T13:59:25.987 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-06T13:59:26.206 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:26.682 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:26.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: Added host smithi134 2024-04-06T13:59:26.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:26.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:26.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:26.683 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 -- 172.21.15.90:0/3760366067 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 msgr2=0x7f35f4105a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 --2- 172.21.15.90:0/3760366067 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4105a80 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f35e4009910 tx=0x7f35e4009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 -- 172.21.15.90:0/3760366067 shutdown_connections 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 --2- 172.21.15.90:0/3760366067 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4105a80 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 -- 172.21.15.90:0/3760366067 >> 172.21.15.90:0/3760366067 conn(0x7f35f4075160 msgr2=0x7f35f4075570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:27.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 -- 172.21.15.90:0/3760366067 shutdown_connections 2024-04-06T13:59:27.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.233+0000 7f35f8d83700 1 -- 172.21.15.90:0/3760366067 wait complete. 2024-04-06T13:59:27.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f8d83700 1 Processor -- start 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f8d83700 1 -- start start 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f8d83700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f8d83700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f35f4195390 con 0x7f35f41056a0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52704/0 (socket says 172.21.15.90:52704) 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f259c700 1 -- 172.21.15.90:0/2776105959 learned_addr learned my addr 172.21.15.90:0/2776105959 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f259c700 1 -- 172.21.15.90:0/2776105959 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35e40095f0 con 0x7f35f41056a0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f259c700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f35e4004f40 tx=0x7f35e4005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f35e401d070 con 0x7f35f41056a0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f35e4004370 con 0x7f35f41056a0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f35f4195590 con 0x7f35f41056a0 2024-04-06T13:59:27.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.234+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f35e4003970 con 0x7f35f41056a0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.235+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35f4195a30 con 0x7f35f41056a0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.235+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f35e4003c30 con 0x7f35f41056a0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.235+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35f40fee30 con 0x7f35f41056a0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.236+0000 7f35e37fe700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 0x7f35dc03ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.236+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f35e404c4a0 con 0x7f35f41056a0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.236+0000 7f35f1d9b700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 0x7f35dc03ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:27.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.236+0000 7f35f1d9b700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 0x7f35dc03ac60 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f35e8006fd0 tx=0x7f35e8006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:27.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.240+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f35e4029950 con 0x7f35f41056a0 2024-04-06T13:59:27.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.439+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f35f4109920 con 0x7f35dc0387a0 2024-04-06T13:59:27.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.441+0000 7f35e37fe700 1 -- 172.21.15.90:0/2776105959 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+240 (secure 0 0 0) 0x7f35f4109920 con 0x7f35dc0387a0 2024-04-06T13:59:27.442 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T13:59:27.442 INFO:teuthology.orchestra.run.smithi090.stdout:[{"addr": "172.21.15.90", "hostname": "smithi090", "labels": [], "status": ""}, {"addr": "172.21.15.105", "hostname": "smithi105", "labels": [], "status": ""}, {"addr": "172.21.15.134", "hostname": "smithi134", "labels": [], "status": ""}] 2024-04-06T13:59:27.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.442+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 msgr2=0x7f35dc03ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.442+0000 7f35f8d83700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 0x7f35dc03ac60 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f35e8006fd0 tx=0x7f35e8006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.442+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 msgr2=0x7f35f4194e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.442+0000 7f35f8d83700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f35e4004f40 tx=0x7f35e4005020 comp rx=0 tx=0).stop 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 shutdown_connections 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f35dc0387a0 0x7f35dc03ac60 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 --2- 172.21.15.90:0/2776105959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f35f41056a0 0x7f35f4194e50 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 >> 172.21.15.90:0/2776105959 conn(0x7f35f4075160 msgr2=0x7f35f40fcb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 shutdown_connections 2024-04-06T13:59:27.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:27.443+0000 7f35f8d83700 1 -- 172.21.15.90:0/2776105959 wait complete. 2024-04-06T13:59:27.821 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-06T13:59:27.821 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-06T13:59:28.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:27 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:28.039 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 -- 172.21.15.90:0/3232667282 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 msgr2=0x7f13e4105a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/3232667282 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4105a80 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f13d4009910 tx=0x7f13d4009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 -- 172.21.15.90:0/3232667282 shutdown_connections 2024-04-06T13:59:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/3232667282 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4105a80 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:29.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 -- 172.21.15.90:0/3232667282 >> 172.21.15.90:0/3232667282 conn(0x7f13e4075160 msgr2=0x7f13e4075570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:29.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 -- 172.21.15.90:0/3232667282 shutdown_connections 2024-04-06T13:59:29.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.025+0000 7f13ea2c7700 1 -- 172.21.15.90:0/3232667282 wait complete. 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13ea2c7700 1 Processor -- start 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13ea2c7700 1 -- start start 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13ea2c7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13ea2c7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13e4068e80 con 0x7f13e41056a0 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13e3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13e3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52720/0 (socket says 172.21.15.90:52720) 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.026+0000 7f13e3fff700 1 -- 172.21.15.90:0/1928733182 learned_addr learned my addr 172.21.15.90:0/1928733182 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13e3fff700 1 -- 172.21.15.90:0/1928733182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13d40095f0 con 0x7f13e41056a0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13e3fff700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f13d4004f40 tx=0x7f13d4005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d401d070 con 0x7f13e41056a0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f13d4004370 con 0x7f13e41056a0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d4003970 con 0x7f13e41056a0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13e4197360 con 0x7f13e41056a0 2024-04-06T13:59:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.027+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13e4197860 con 0x7f13e41056a0 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.028+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13e40fee30 con 0x7f13e41056a0 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.028+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f13d4003be0 con 0x7f13e41056a0 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.028+0000 7f13e17fa700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 0x7f13cc03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.029+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f13d404c880 con 0x7f13e41056a0 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.029+0000 7f13e37fe700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 0x7f13cc03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:29.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.029+0000 7f13e37fe700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 0x7f13cc03acb0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f13d8006fd0 tx=0x7f13d8006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:29.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.033+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f13d40044e0 con 0x7f13e41056a0 2024-04-06T13:59:29.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:29.235+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f13e4100d00 con 0x7f13e41056a0 2024-04-06T13:59:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:28 smithi090 ceph-mon[43420]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T13:59:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:28 smithi090 ceph-mon[43420]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:30.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.129+0000 7f13e17fa700 1 -- 172.21.15.90:0/1928733182 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) v1 ==== 124+0+0 (secure 0 0 0) 0x7f13d4026020 con 0x7f13e41056a0 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 msgr2=0x7f13cc03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 0x7f13cc03acb0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f13d8006fd0 tx=0x7f13d8006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 msgr2=0x7f13e4196dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f13d4004f40 tx=0x7f13d4005020 comp rx=0 tx=0).stop 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 shutdown_connections 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f13cc0387f0 0x7f13cc03acb0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 --2- 172.21.15.90:0/1928733182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f13e41056a0 0x7f13e4196dc0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 >> 172.21.15.90:0/1928733182 conn(0x7f13e4075160 msgr2=0x7f13e40fcb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 shutdown_connections 2024-04-06T13:59:30.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:30.132+0000 7f13ea2c7700 1 -- 172.21.15.90:0/1928733182 wait complete. 2024-04-06T13:59:30.134 INFO:teuthology.orchestra.run.smithi090.stderr:adjusted tunables profile to default 2024-04-06T13:59:30.494 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.494 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.495 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:30.562 INFO:tasks.cephadm:Adding mon.a on smithi090 2024-04-06T13:59:30.562 INFO:tasks.cephadm:Adding mon.b on smithi105 2024-04-06T13:59:30.563 INFO:tasks.cephadm:Adding mon.c on smithi134 2024-04-06T13:59:30.563 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch apply mon '3;smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c' 2024-04-06T13:59:30.810 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:31 smithi090 ceph-mon[43420]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:31 smithi090 ceph-mon[43420]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T13:59:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:31 smithi090 ceph-mon[43420]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-06T13:59:31.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:31 smithi090 ceph-mon[43420]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- 172.21.15.134:0/2678827555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 msgr2=0x7f49b8105a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 --2- 172.21.15.134:0/2678827555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8105a80 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f49ac009910 tx=0x7f49ac009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- 172.21.15.134:0/2678827555 shutdown_connections 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 --2- 172.21.15.134:0/2678827555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8105a80 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- 172.21.15.134:0/2678827555 >> 172.21.15.134:0/2678827555 conn(0x7f49b8075160 msgr2=0x7f49b8075570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- 172.21.15.134:0/2678827555 shutdown_connections 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- 172.21.15.134:0/2678827555 wait complete. 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 Processor -- start 2024-04-06T13:59:31.674 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.672+0000 7f49bf036700 1 -- start start 2024-04-06T13:59:31.675 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bf036700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:31.675 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bf036700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49b8068e80 con 0x7f49b81056a0 2024-04-06T13:59:31.675 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bcdd2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:31.675 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bcdd2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.134:38156/0 (socket says 172.21.15.134:38156) 2024-04-06T13:59:31.675 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bcdd2700 1 -- 172.21.15.134:0/2721025336 learned_addr learned my addr 172.21.15.134:0/2721025336 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T13:59:31.676 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.673+0000 7f49bcdd2700 1 -- 172.21.15.134:0/2721025336 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f49ac0095f0 con 0x7f49b81056a0 2024-04-06T13:59:31.676 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49bcdd2700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f49ac004f40 tx=0x7f49ac005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:31.676 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f49ac01d070 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f49b8197360 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f49ac004370 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f49ac003970 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.674+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49b8197860 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.675+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f49b80fee30 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.675+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f49ac003c30 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.675+0000 7f49b5ffb700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 0x7f49a803ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.675+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f49ac04c420 con 0x7f49b81056a0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.676+0000 7f49b7fff700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 0x7f49a803ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:31.677 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.676+0000 7f49b7fff700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 0x7f49a803ac60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f49a4006fd0 tx=0x7f49a4006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:31.681 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.681+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f49ac0044e0 con 0x7f49b81056a0 2024-04-06T13:59:31.883 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.882+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c", "target": ["mon-mgr", ""]}) v1 -- 0x7f49b8197b80 con 0x7f49a80387a0 2024-04-06T13:59:31.890 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.889+0000 7f49b5ffb700 1 -- 172.21.15.134:0/2721025336 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f49b8197b80 con 0x7f49a80387a0 2024-04-06T13:59:31.890 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mon update... 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 msgr2=0x7f49a803ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 0x7f49a803ac60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f49a4006fd0 tx=0x7f49a4006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 msgr2=0x7f49b8196dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f49ac004f40 tx=0x7f49ac005020 comp rx=0 tx=0).stop 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 shutdown_connections 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f49a80387a0 0x7f49a803ac60 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 --2- 172.21.15.134:0/2721025336 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49b81056a0 0x7f49b8196dc0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 >> 172.21.15.134:0/2721025336 conn(0x7f49b8075160 msgr2=0x7f49b80fcb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 shutdown_connections 2024-04-06T13:59:31.892 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:31.891+0000 7f49bf036700 1 -- 172.21.15.134:0/2721025336 wait complete. 2024-04-06T13:59:32.238 DEBUG:teuthology.orchestra.run.smithi105:mon.b> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.b.service 2024-04-06T13:59:32.241 DEBUG:teuthology.orchestra.run.smithi134:mon.c> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.c.service 2024-04-06T13:59:32.243 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-06T13:59:32.243 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-06T13:59:32.272 INFO:journalctl@ceph.mon.c.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T13:59:32.277 INFO:journalctl@ceph.mon.b.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T13:59:32.665 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: Saving service mon spec with placement smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c;count:3 2024-04-06T13:59:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: Deploying daemon mon.c on smithi134 2024-04-06T13:59:33.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:32 smithi090 ceph-mon[43420]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:34.149 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.146+0000 7fedfddb6700 1 -- 172.21.15.134:0/2665338487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 msgr2=0x7fedf8106aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.146+0000 7fedfddb6700 1 --2- 172.21.15.134:0/2665338487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8106aa0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fede8009910 tx=0x7fede8009c20 comp rx=0 tx=0).stop 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 -- 172.21.15.134:0/2665338487 shutdown_connections 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 --2- 172.21.15.134:0/2665338487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8106aa0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 -- 172.21.15.134:0/2665338487 >> 172.21.15.134:0/2665338487 conn(0x7fedf80fc1b0 msgr2=0x7fedf80fe5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 -- 172.21.15.134:0/2665338487 shutdown_connections 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 -- 172.21.15.134:0/2665338487 wait complete. 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 Processor -- start 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.147+0000 7fedfddb6700 1 -- start start 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedfddb6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedfddb6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fedf81964a0 con 0x7fedf81066c0 2024-04-06T13:59:34.150 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.134:38172/0 (socket says 172.21.15.134:38172) 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf77fe700 1 -- 172.21.15.134:0/3591460589 learned_addr learned my addr 172.21.15.134:0/3591460589 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf77fe700 1 -- 172.21.15.134:0/3591460589 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fede80095f0 con 0x7fedf81066c0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf77fe700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fede8004f40 tx=0x7fede8005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fede801d070 con 0x7fedf81066c0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.148+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fedf8196700 con 0x7fedf81066c0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.149+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fedf8196b80 con 0x7fedf81066c0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.149+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fede800bbb0 con 0x7fedf81066c0 2024-04-06T13:59:34.151 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.149+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fede8003680 con 0x7fedf81066c0 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.149+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fede40052f0 con 0x7fedf81066c0 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.149+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7fede800b6f0 con 0x7fedf81066c0 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.150+0000 7fedf4ff9700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 0x7fede003acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.150+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fede804c350 con 0x7fedf81066c0 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.150+0000 7fedf6ffd700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 0x7fede003acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:34.152 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.150+0000 7fedf6ffd700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 0x7fede003acb0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fedec006fd0 tx=0x7fedec006e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:34.157 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.155+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fede80037e0 con 0x7fedf81066c0 2024-04-06T13:59:34.464 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.462+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fede4005160 con 0x7fedf81066c0 2024-04-06T13:59:34.464 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.462+0000 7fedf4ff9700 1 -- 172.21.15.134:0/3591460589 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+736 (secure 0 0 0) 0x7fede8026070 con 0x7fedf81066c0 2024-04-06T13:59:34.464 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:59:34.464 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":1,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","modified":"2024-04-06T13:58:11.343476Z","created":"2024-04-06T13:58:11.343476Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3300","nonce":0},{"type":"v1","addr":"172.21.15.90:6789","nonce":0}]},"addr":"172.21.15.90:6789/0","public_addr":"172.21.15.90:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-06T13:59:34.466 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 msgr2=0x7fede003acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:34.466 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 0x7fede003acb0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fedec006fd0 tx=0x7fedec006e40 comp rx=0 tx=0).stop 2024-04-06T13:59:34.466 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 msgr2=0x7fedf8195f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:34.466 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fede8004f40 tx=0x7fede8005020 comp rx=0 tx=0).stop 2024-04-06T13:59:34.466 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 shutdown_connections 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fede00387f0 0x7fede003acb0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 --2- 172.21.15.134:0/3591460589 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fedf81066c0 0x7fedf8195f60 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 >> 172.21.15.134:0/3591460589 conn(0x7fedf80fc1b0 msgr2=0x7fedf81082f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 shutdown_connections 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:34.465+0000 7fedfddb6700 1 -- 172.21.15.134:0/3591460589 wait complete. 2024-04-06T13:59:34.467 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 1 2024-04-06T13:59:34.489 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 systemd[1]: Starting Ceph mon.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 bash[47511]: c4f1b50895df50604b0049c719b32929458468cf486897e1f594f57072c8ca7f 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 systemd[1]: Started Ceph mon.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 ceph-mon[47549]: set uid:gid to 167:167 (ceph:ceph) 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 ceph-mon[47549]: ceph version 18.2.2-1086-g052adcad (052adcadd0534eddb0d73c5afbc083a84cd9dc46) reef (stable), process ceph-mon, pid 7 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:34 smithi134 ceph-mon[47549]: pidfile_write: ignore empty --pid-file 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: load: jerasure load: lrc 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: RocksDB version: 7.9.2 2024-04-06T13:59:35.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Git sha 0 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Compile date 2024-04-03 02:24:10 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: DB SUMMARY 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: DB Session ID: D1E8XA35E95Y3YPFIX19 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: CURRENT file: CURRENT 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: IDENTITY file: IDENTITY 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.error_if_exists: 0 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.create_if_missing: 0 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.paranoid_checks: 1 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-06T13:59:35.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.env: 0x55ea180de7a0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.fs: PosixFileSystem 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.info_log: 0x55ea19b894e0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_file_opening_threads: 16 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.statistics: (nil) 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.use_fsync: 0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_log_file_size: 0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.keep_log_file_num: 1000 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.recycle_log_file_num: 0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_fallocate: 1 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_mmap_reads: 0 2024-04-06T13:59:35.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_mmap_writes: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.use_direct_reads: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.create_missing_column_families: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.db_log_dir: 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.wal_dir: 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-06T13:59:35.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.advise_random_on_open: 1 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.db_write_buffer_size: 0 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.write_buffer_manager: 0x55ea18e20320 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.rate_limiter: (nil) 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.wal_recovery_mode: 2 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enable_thread_tracking: 0 2024-04-06T13:59:35.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enable_pipelined_write: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.unordered_write: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.row_cache: None 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.wal_filter: None 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_ingest_behind: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.two_write_queues: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.manual_wal_flush: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.wal_compression: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.atomic_flush: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-06T13:59:35.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.log_readahead_size: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.best_efforts_recovery: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.allow_data_in_errors: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.db_host_id: __hostname__ 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_background_jobs: 2 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_background_compactions: -1 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_subcompactions: 1 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-06T13:59:35.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_total_wal_size: 0 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_open_files: -1 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bytes_per_sync: 0 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_readahead_size: 0 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_background_flushes: -1 2024-04-06T13:59:35.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Compression algorithms supported: 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kZSTD supported: 0 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kXpressCompression supported: 0 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kLZ4HCCompression supported: 1 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kZlibCompression supported: 1 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kSnappyCompression supported: 1 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kLZ4Compression supported: 1 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: kBZip2Compression supported: 0 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-04-06T13:59:35.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.merge_operator: 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_filter: None 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_filter_factory: None 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.sst_partitioner_factory: None 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ea19b895e0) 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout: cache_index_and_filter_blocks: 1 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout: pin_top_level_index_and_filter: 1 2024-04-06T13:59:35.404 INFO:journalctl@ceph.mon.c.smithi134.stdout: index_type: 0 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: data_block_index_type: 0 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: index_shortening: 1 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: checksum: 4 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: no_block_cache: 0 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_cache: 0x55ea18ea11f0 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_cache_name: BinnedLRUCache 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_cache_options: 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: capacity : 536870912 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: num_shard_bits : 4 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: strict_capacity_limit : 0 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: high_pri_pool_ratio: 0.000 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_cache_compressed: (nil) 2024-04-06T13:59:35.405 INFO:journalctl@ceph.mon.c.smithi134.stdout: persistent_cache: (nil) 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_size: 4096 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_size_deviation: 10 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_restart_interval: 16 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: index_block_restart_interval: 1 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: metadata_block_size: 4096 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: partition_filters: 0 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: use_delta_encoding: 1 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: filter_policy: bloomfilter 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: whole_key_filtering: 1 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: verify_compression: 0 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: read_amp_bytes_per_bit: 0 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: format_version: 5 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: enable_index_compression: 1 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: block_align: 0 2024-04-06T13:59:35.406 INFO:journalctl@ceph.mon.c.smithi134.stdout: max_auto_readahead_size: 262144 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout: prepopulate_block_cache: 0 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout: initial_auto_readahead_size: 8192 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout: num_file_reads_for_auto_readahead: 2 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.write_buffer_size: 33554432 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_write_buffer_number: 2 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression: NoCompression 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression: Disabled 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.prefix_extractor: nullptr 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.num_levels: 7 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-06T13:59:35.407 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.level: 32767 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.strategy: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-06T13:59:35.408 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.enabled: false 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.target_file_size_base: 67108864 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-06T13:59:35.409 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.arena_block_size: 1048576 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-06T13:59:35.410 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.disable_auto_compactions: 0 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.table_properties_collectors: 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.inplace_update_support: 0 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-06T13:59:35.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.bloom_locality: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.max_successive_merges: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.paranoid_file_checks: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.force_consistency_checks: 1 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.report_bg_io_stats: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.ttl: 2592000 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enable_blob_files: false 2024-04-06T13:59:35.412 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.min_blob_size: 0 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_file_size: 268435456 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.blob_file_starting_level: 0 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7c495936-6fdd-4b79-9fa1-2ffdef4c041c 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411975005384, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-04-06T13:59:35.413 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411975005793, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712411975, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7c495936-6fdd-4b79-9fa1-2ffdef4c041c", "db_session_id": "D1E8XA35E95Y3YPFIX19", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411975005861, "job": 1, "event": "recovery_finished"} 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ea18f3e000 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: DB pointer 0x55ea18f2c000 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: ** DB Stats ** 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:35.414 INFO:journalctl@ceph.mon.c.smithi134.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: ** Compaction Stats [default] ** 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: ** Compaction Stats [default] ** 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.415 INFO:journalctl@ceph.mon.c.smithi134.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Block cache BinnedLRUCache@0x55ea18ea11f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-06T13:59:35.416 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-06T13:59:35.417 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: using public_addr v2:172.21.15.134:0/0 -> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] 2024-04-06T13:59:35.417 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: starting mon.c rank -1 at public addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] at bind addrs [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:35.417 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(???) e0 preinit fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).mds e1 new map 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: e1 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: legacy client fscid: -1 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout: No filesystems configured 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-06T13:59:35.418 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:35.419 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:35.420 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Saving service mon spec with placement smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c;count:3 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: Deploying daemon mon.c on smithi134 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:35.421 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:35 smithi134 ceph-mon[47549]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-06T13:59:35.893 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-06T13:59:35.893 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-06T13:59:36.125 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 -- 172.21.15.134:0/2357053316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bb00035c0 msgr2=0x7f8bb0005a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/2357053316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bb00035c0 0x7f8bb0005a50 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f8bc4077f80 tx=0x7f8bac013e90 comp rx=0 tx=0).stop 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 -- 172.21.15.134:0/2357053316 shutdown_connections 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/2357053316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bb00035c0 0x7f8bb0005a50 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 -- 172.21.15.134:0/2357053316 >> 172.21.15.134:0/2357053316 conn(0x7f8bc4076cb0 msgr2=0x7f8bc40770c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:40.239 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 -- 172.21.15.134:0/2357053316 shutdown_connections 2024-04-06T13:59:40.240 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.238+0000 7f8bc88ac700 1 -- 172.21.15.134:0/2357053316 wait complete. 2024-04-06T13:59:40.240 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.239+0000 7f8bc88ac700 1 Processor -- start 2024-04-06T13:59:40.240 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.239+0000 7f8bc88ac700 1 -- start start 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc88ac700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc88ac700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 0x7f8bc4197850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc88ac700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8bc419ff00 con 0x7f8bc41973d0 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc88ac700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8bc41a0040 con 0x7f8bb00035c0 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:35628/0 (socket says 172.21.15.134:35628) 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 -- 172.21.15.134:0/1623155809 learned_addr learned my addr 172.21.15.134:0/1623155809 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 -- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 msgr2=0x7f8bc4196e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 -- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 msgr2=0x7f8bc4196e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-06T13:59:40.242 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc1d9b700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 0x7f8bc4197850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc259c700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc1d9b700 1 -- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 msgr2=0x7f8bc4196e90 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc1d9b700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.240+0000 7f8bc1d9b700 1 -- 172.21.15.134:0/1623155809 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8bac0138e0 con 0x7f8bc41973d0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bc1d9b700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 0x7f8bc4197850 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8bb400b500 tx=0x7f8bb400b8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7f8bb4011630 con 0x7f8bc41973d0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8bb4011c50 con 0x7f8bc41973d0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8bc41a0180 con 0x7f8bc41973d0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7f8bb4010cc0 con 0x7f8bc41973d0 2024-04-06T13:59:40.243 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.241+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8bc41a06d0 con 0x7f8bc41973d0 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.242+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f8bb401a410 con 0x7f8bc41973d0 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.242+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8bc4103980 con 0x7f8bc41973d0 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.242+0000 7f8bbb7fe700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 0x7f8ba403aa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.242+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f8bb404a750 con 0x7f8bc41973d0 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.243+0000 7f8bc259c700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 0x7f8ba403aa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:40.244 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.243+0000 7f8bc259c700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 0x7f8ba403aa90 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f8bac00f470 tx=0x7f8bac00f3c0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:40.248 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.248+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8bb4010e20 con 0x7f8bc41973d0 2024-04-06T13:59:40.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:40.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: mon.a calling monitor election 2024-04-06T13:59:40.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: mon.c calling monitor election 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: fsmap 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: mgrmap e13: x(active, since 35s) 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: overall HEALTH_OK 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:40.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:40.524 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.524+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8bc4100fc0 con 0x7f8bc41973d0 2024-04-06T13:59:40.525 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.524+0000 7f8bbb7fe700 1 -- 172.21.15.134:0/1623155809 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) v1 ==== 95+0+1006 (secure 0 0 0) 0x7f8bb40186e0 con 0x7f8bc41973d0 2024-04-06T13:59:40.525 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:59:40.525 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":2,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","modified":"2024-04-06T13:59:35.037009Z","created":"2024-04-06T13:58:11.343476Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3300","nonce":0},{"type":"v1","addr":"172.21.15.90:6789","nonce":0}]},"addr":"172.21.15.90:6789/0","public_addr":"172.21.15.90:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-06T13:59:40.527 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 msgr2=0x7f8ba403aa90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:40.527 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 0x7f8ba403aa90 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f8bac00f470 tx=0x7f8bac00f3c0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.527 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 msgr2=0x7f8bc4197850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:40.527 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 0x7f8bc4197850 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8bb400b500 tx=0x7f8bb400b8c0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.527 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 shutdown_connections 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8ba40385d0 0x7f8ba403aa90 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8bb00035c0 0x7f8bc4196e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 --2- 172.21.15.134:0/1623155809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8bc41973d0 0x7f8bc4197850 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 >> 172.21.15.134:0/1623155809 conn(0x7f8bc4076cb0 msgr2=0x7f8bc40fd230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 shutdown_connections 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:40.526+0000 7f8bc88ac700 1 -- 172.21.15.134:0/1623155809 wait complete. 2024-04-06T13:59:40.528 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 2 2024-04-06T13:59:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: mon.a calling monitor election 2024-04-06T13:59:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: mon.c calling monitor election 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: fsmap 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: mgrmap e13: x(active, since 35s) 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: overall HEALTH_OK 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:40.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:40.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:41.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:41 smithi134 ceph-mon[47549]: Deploying daemon mon.b on smithi105 2024-04-06T13:59:41.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:41 smithi134 ceph-mon[47549]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:41.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:41 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/1623155809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-06T13:59:41.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:41 smithi090 ceph-mon[43420]: Deploying daemon mon.b on smithi105 2024-04-06T13:59:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:41 smithi090 ceph-mon[43420]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:41.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:41 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/1623155809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-06T13:59:41.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:41.897 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-06T13:59:41.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-06T13:59:42.129 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 systemd[1]: Started Ceph mon.b for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411982650679, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712411982, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "285b2f3c-bbe2-48c6-a0b5-6c5a78a55803", "db_session_id": "BLPMUJ7XPD1DS334XLUS", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712411982650768, "job": 1, "event": "recovery_finished"} 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a5f2fae000 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: DB pointer 0x55a5f2f9c000 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-06T13:59:42.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** DB Stats ** 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-06T13:59:42.999 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-06T13:59:43.000 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache BinnedLRUCache@0x55a5f2f111f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: using public_addr v2:172.21.15.105:0/0 -> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] 2024-04-06T13:59:43.001 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: starting mon.b rank -1 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(???) e0 preinit fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).mds e1 new map 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: e1 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: legacy client fscid: -1 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout: No filesystems configured 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-06T13:59:43.002 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.003 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi134", "name": "osd_memory_target"}]: dispatch 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1928733182' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-06T13:59:43.004 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Saving service mon spec with placement smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c;count:3 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Deploying daemon mon.c on smithi134 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.005 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.a calling monitor election 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.c calling monitor election 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: monmap e2: 2 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:43.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: fsmap 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mgrmap e13: x(active, since 35s) 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: overall HEALTH_OK 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: Deploying daemon mon.b on smithi105 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/1623155809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:43.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:42 smithi105 ceph-mon[46969]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: mon.a calling monitor election 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: mon.c calling monitor election 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:48.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: fsmap 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: mgrmap e13: x(active, since 43s) 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-06T13:59:48.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: mon.b (rank 2) addr [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] is down (out of quorum) 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:48.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: mon.a calling monitor election 2024-04-06T13:59:48.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: mon.c calling monitor election 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: fsmap 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:48.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: mgrmap e13: x(active, since 43s) 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: mon.b (rank 2) addr [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] is down (out of quorum) 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:48.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.019 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 -- 172.21.15.134:0/2676159513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81c80062a0 msgr2=0x7f81c8008710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:49.019 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/2676159513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81c80062a0 0x7f81c8008710 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f81e0100790 tx=0x7f81d000fe90 comp rx=0 tx=0).stop 2024-04-06T13:59:49.020 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 -- 172.21.15.134:0/2676159513 shutdown_connections 2024-04-06T13:59:49.020 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/2676159513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81c80062a0 0x7f81c8008710 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.020 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 -- 172.21.15.134:0/2676159513 >> 172.21.15.134:0/2676159513 conn(0x7f81e0076380 msgr2=0x7f81e00fd6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:49.020 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 -- 172.21.15.134:0/2676159513 shutdown_connections 2024-04-06T13:59:49.020 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.018+0000 7f81e5ed4700 1 -- 172.21.15.134:0/2676159513 wait complete. 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 Processor -- start 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 -- start start 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 0x7f81e019b540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81e0196e90 con 0x7f81e019b090 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81e01a2e00 con 0x7f81c80062a0 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81e5ed4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f81e01a2f40 con 0x7f81e01967a0 2024-04-06T13:59:49.023 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81deffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81deffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:38382/0 (socket says 172.21.15.134:38382) 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81deffd700 1 -- 172.21.15.134:0/1094389859 learned_addr learned my addr 172.21.15.134:0/1094389859 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.019+0000 7f81df7fe700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81deffd700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 msgr2=0x7f81e019ab50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 13 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81deffd700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 msgr2=0x7f81e019ab50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81deffd700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81df7fe700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 msgr2=0x7f81e0196260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81df7fe700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 msgr2=0x7f81e0196260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 0x7f81e019b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81deffd700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81df7fe700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81df7fe700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 msgr2=0x7f81e019ab50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:59:49.024 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 msgr2=0x7f81e0196260 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 -- 172.21.15.134:0/1094389859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f81d000f8e0 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dffff700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 0x7f81e019b540 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f81d800bd50 tx=0x7f81d800be30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81d800cd10 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f81d800ce70 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f81d8012840 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f81e01a3220 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.020+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f81e01a36d0 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.021+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f81d80129c0 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.021+0000 7f81dcff9700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 0x7f81cc03b130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.021+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f81d804aff0 con 0x7f81e019b090 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.022+0000 7f81df7fe700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 0x7f81cc03b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:49.025 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.022+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f81e0100e10 con 0x7f81e019b090 2024-04-06T13:59:49.026 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.022+0000 7f81df7fe700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 0x7f81cc03b130 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f81d0005b40 tx=0x7f81d0005a90 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:49.028 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.027+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f81d8019070 con 0x7f81e019b090 2024-04-06T13:59:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:49.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:49.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.312 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.311+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f81e0197570 con 0x7f81e019b090 2024-04-06T13:59:49.313 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.312+0000 7f81dcff9700 1 -- 172.21.15.134:0/1094389859 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) v1 ==== 95+0+1274 (secure 0 0 0) 0x7f81d801d3b0 con 0x7f81e019b090 2024-04-06T13:59:49.313 INFO:teuthology.orchestra.run.smithi134.stdout: 2024-04-06T13:59:49.313 INFO:teuthology.orchestra.run.smithi134.stdout:{"epoch":3,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","modified":"2024-04-06T13:59:42.680967Z","created":"2024-04-06T13:58:11.343476Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:3300","nonce":0},{"type":"v1","addr":"172.21.15.90:6789","nonce":0}]},"addr":"172.21.15.90:6789/0","public_addr":"172.21.15.90:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:3300","nonce":0},{"type":"v1","addr":"172.21.15.134:6789","nonce":0}]},"addr":"172.21.15.134:6789/0","public_addr":"172.21.15.134:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 msgr2=0x7f81cc03b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 0x7f81cc03b130 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f81d0005b40 tx=0x7f81d0005a90 comp rx=0 tx=0).stop 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 msgr2=0x7f81e019b540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 0x7f81e019b540 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f81d800bd50 tx=0x7f81d800be30 comp rx=0 tx=0).stop 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 shutdown_connections 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f81cc038c70 0x7f81cc03b130 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f81e019b090 0x7f81e019b540 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81c80062a0 0x7f81e0196260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.315 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 --2- 172.21.15.134:0/1094389859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f81e01967a0 0x7f81e019ab50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:49.316 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 >> 172.21.15.134:0/1094389859 conn(0x7f81e0076380 msgr2=0x7f81e0104a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:49.316 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 shutdown_connections 2024-04-06T13:59:49.316 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:49.314+0000 7f81e5ed4700 1 -- 172.21.15.134:0/1094389859 wait complete. 2024-04-06T13:59:49.316 INFO:teuthology.orchestra.run.smithi134.stderr:dumped monmap epoch 3 2024-04-06T13:59:49.671 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-06T13:59:49.671 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: mon.a calling monitor election 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: mon.c calling monitor election 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: fsmap 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: mgrmap e13: x(active, since 43s) 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-06T13:59:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: mon.b (rank 2) addr [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] is down (out of quorum) 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:50.194 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mon.b calling monitor election 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mon.b calling monitor election 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mon.a calling monitor election 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mon.c calling monitor election 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-06T13:59:50.572 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: fsmap 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: mgrmap e13: x(active, since 45s) 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: Cluster is now healthy 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: overall HEALTH_OK 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:50.573 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:50.639 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mon.b calling monitor election 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mon.b calling monitor election 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mon.a calling monitor election 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mon.c calling monitor election 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: fsmap 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: mgrmap e13: x(active, since 45s) 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: Cluster is now healthy 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: overall HEALTH_OK 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.640 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.641 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:50.641 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:50.641 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mon.b calling monitor election 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mon.b calling monitor election 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mon.a calling monitor election 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mon.c calling monitor election 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: monmap e3: 3 mons at {a=[v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: fsmap 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: osdmap e4: 0 total, 0 up, 0 in 2024-04-06T13:59:50.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: mgrmap e13: x(active, since 45s) 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: Cluster is now healthy 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: overall HEALTH_OK 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:50.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:51.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 -- 172.21.15.90:0/1648209571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed41006a0 msgr2=0x7efed410cb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 --2- 172.21.15.90:0/1648209571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed41006a0 0x7efed410cb50 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7efecc00b0a0 tx=0x7efecc00b3b0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 -- 172.21.15.90:0/1648209571 shutdown_connections 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 --2- 172.21.15.90:0/1648209571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed41006a0 0x7efed410cb50 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 --2- 172.21.15.90:0/1648209571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed4105d10 0x7efed41060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 --2- 172.21.15.90:0/1648209571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed40ffce0 0x7efed4100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 -- 172.21.15.90:0/1648209571 >> 172.21.15.90:0/1648209571 conn(0x7efed4075d90 msgr2=0x7efed40761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 -- 172.21.15.90:0/1648209571 shutdown_connections 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.176+0000 7efed8b9e700 1 -- 172.21.15.90:0/1648209571 wait complete. 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 Processor -- start 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 -- start start 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:51.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 0x7efed4197850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed4105d10 0x7efed419bc00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efed419c210 con 0x7efed40ffce0 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efed419c380 con 0x7efed41006a0 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.177+0000 7efed8b9e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efed419c540 con 0x7efed4105d10 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52630/0 (socket says 172.21.15.90:52630) 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 -- 172.21.15.90:0/2496161729 learned_addr learned my addr 172.21.15.90:0/2496161729 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed1d9b700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 0x7efed4197850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed2d9d700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed4105d10 0x7efed419bc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 -- 172.21.15.90:0/2496161729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed4105d10 msgr2=0x7efed419bc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed4105d10 0x7efed419bc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 -- 172.21.15.90:0/2496161729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 msgr2=0x7efed4197850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 0x7efed4197850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 -- 172.21.15.90:0/2496161729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efecc009d00 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed1d9b700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 0x7efed4197850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed259c700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7efec400e9a0 tx=0x7efec400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec4009790 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efed419c820 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efec400cbd0 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efec4018680 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.178+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efed41a4080 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.179+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7efec40187e0 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.179+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efed41013c0 con 0x7efed40ffce0 2024-04-06T13:59:51.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.179+0000 7efecb7fe700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 0x7efec003ae90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:51.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.179+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7efec4014070 con 0x7efed40ffce0 2024-04-06T13:59:51.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.179+0000 7efed1d9b700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 0x7efec003ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:51.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.180+0000 7efed1d9b700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 0x7efec003ae90 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7efed4198870 tx=0x7efebc00b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:51.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.184+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efec40152d0 con 0x7efed40ffce0 2024-04-06T13:59:51.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.377+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7efed419c9b0 con 0x7efed40ffce0 2024-04-06T13:59:51.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.377+0000 7efecb7fe700 1 -- 172.21.15.90:0/2496161729 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) v1 ==== 76+0+275 (secure 0 0 0) 0x7efed419c9b0 con 0x7efed40ffce0 2024-04-06T13:59:51.377 INFO:teuthology.orchestra.run.smithi090.stdout:# minimal ceph.conf for 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:51.377 INFO:teuthology.orchestra.run.smithi090.stdout:[global] 2024-04-06T13:59:51.378 INFO:teuthology.orchestra.run.smithi090.stdout: fsid = 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T13:59:51.378 INFO:teuthology.orchestra.run.smithi090.stdout: mon_host = [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] 2024-04-06T13:59:51.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 msgr2=0x7efec003ae90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 0x7efec003ae90 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7efed4198870 tx=0x7efebc00b410 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 msgr2=0x7efed4197310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7efec400e9a0 tx=0x7efec400ecb0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 shutdown_connections 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efec00389d0 0x7efec003ae90 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efed4105d10 0x7efed419bc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efed40ffce0 0x7efed4197310 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 --2- 172.21.15.90:0/2496161729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efed41006a0 0x7efed4197850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.379+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 >> 172.21.15.90:0/2496161729 conn(0x7efed4075d90 msgr2=0x7efed4068940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.380+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 shutdown_connections 2024-04-06T13:59:51.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:51.380+0000 7efed8b9e700 1 -- 172.21.15.90:0/2496161729 wait complete. 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: Reconfiguring mon.b (monmap changed)... 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: Reconfiguring daemon mon.b on smithi105 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.637 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.638 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:51.638 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:51.638 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:51.668 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:51.668 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: Reconfiguring mon.b (monmap changed)... 2024-04-06T13:59:51.668 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: Reconfiguring daemon mon.b on smithi105 2024-04-06T13:59:51.668 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:51.668 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:51.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:51.669 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: Reconfiguring mon.b (monmap changed)... 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: Reconfiguring daemon mon.b on smithi105 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:51.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-06T13:59:51.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-06T13:59:51.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:51.761 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-06T13:59:51.761 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:51.761 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.conf 2024-04-06T13:59:51.790 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:51.790 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:51.857 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:59:51.857 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.conf 2024-04-06T13:59:51.895 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:59:51.895 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:51.966 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:59:51.966 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.conf 2024-04-06T13:59:51.999 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:59:51.999 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-06T13:59:52.093 INFO:tasks.cephadm:Adding mgr.x on smithi090 2024-04-06T13:59:52.093 INFO:tasks.cephadm:Adding mgr.y on smithi105 2024-04-06T13:59:52.093 INFO:tasks.cephadm:Adding mgr.z on smithi134 2024-04-06T13:59:52.093 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch apply mgr '3;smithi090=x;smithi105=y;smithi134=z' 2024-04-06T13:59:52.344 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T13:59:52.588 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: Reconfiguring mon.c (monmap changed)... 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: Reconfiguring daemon mon.c on smithi134 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2496161729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:52.589 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: Reconfiguring mon.c (monmap changed)... 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: Reconfiguring daemon mon.c on smithi134 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2496161729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:52.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: Reconfiguring mon.c (monmap changed)... 2024-04-06T13:59:52.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: Reconfiguring daemon mon.c on smithi134 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2496161729' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 -- 172.21.15.134:0/839576524 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 msgr2=0x7f3898101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 --2- 172.21.15.134:0/839576524 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f3898101370 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3888009990 tx=0x7f3888009ca0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 -- 172.21.15.134:0/839576524 shutdown_connections 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 --2- 172.21.15.134:0/839576524 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f38981018b0 0x7f3898108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 --2- 172.21.15.134:0/839576524 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f38980ffd10 0x7f3898100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 --2- 172.21.15.134:0/839576524 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f3898101370 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 -- 172.21.15.134:0/839576524 >> 172.21.15.134:0/839576524 conn(0x7f38980fb2b0 msgr2=0x7f38980fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 -- 172.21.15.134:0/839576524 shutdown_connections 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.202+0000 7f389dc49700 1 -- 172.21.15.134:0/839576524 wait complete. 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 Processor -- start 2024-04-06T13:59:53.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 -- start start 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f38980ffd10 0x7f3898196d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f38981018b0 0x7f389819e270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f389819e840 con 0x7f38981018b0 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f389819e980 con 0x7f38980ffd10 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f389dc49700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f389819eb40 con 0x7f3898100f10 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f3896ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f3896ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:38408/0 (socket says 172.21.15.134:38408) 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.203+0000 7f3896ffd700 1 -- 172.21.15.134:0/742882318 learned_addr learned my addr 172.21.15.134:0/742882318 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 -- 172.21.15.134:0/742882318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f38980ffd10 msgr2=0x7f3898196d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f38980ffd10 0x7f3898196d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 -- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f38981018b0 msgr2=0x7f389819e270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f38981018b0 0x7f389819e270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.205 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 -- 172.21.15.134:0/742882318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f38880095f0 con 0x7f3898100f10 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3896ffd700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f388800b5c0 tx=0x7f388800f7b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f388801d070 con 0x7f3898100f10 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f388800fd60 con 0x7f3898100f10 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f389819edc0 con 0x7f3898100f10 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3888017670 con 0x7f3898100f10 2024-04-06T13:59:53.206 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.204+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f389819f180 con 0x7f3898100f10 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.205+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f389804eaf0 con 0x7f3898100f10 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.205+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f3888003960 con 0x7f3898100f10 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.206+0000 7f3894ff9700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 0x7f388003ae90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.206+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f388804c7a0 con 0x7f3898100f10 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.206+0000 7f38977fe700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 0x7f388003ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:53.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.207+0000 7f38977fe700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 0x7f388003ae90 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f38981981d0 tx=0x7f388c005d20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:53.210 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.210+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3888003c10 con 0x7f3898100f10 2024-04-06T13:59:53.414 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.414+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi090=x;smithi105=y;smithi134=z", "target": ["mon-mgr", ""]}) v1 -- 0x7f389804eb10 con 0x7f38800389d0 2024-04-06T13:59:53.420 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.420+0000 7f3894ff9700 1 -- 172.21.15.134:0/742882318 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f388c00a640 con 0x7f38800389d0 2024-04-06T13:59:53.420 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mgr update... 2024-04-06T13:59:53.422 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 msgr2=0x7f388003ae90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:53.422 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 0x7f388003ae90 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f38981981d0 tx=0x7f388c005d20 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 msgr2=0x7f38981972d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f388800b5c0 tx=0x7f388800f7b0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 shutdown_connections 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f38800389d0 0x7f388003ae90 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f38981018b0 0x7f389819e270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f38980ffd10 0x7f3898196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 --2- 172.21.15.134:0/742882318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3898100f10 0x7f38981972d0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 >> 172.21.15.134:0/742882318 conn(0x7f38980fb2b0 msgr2=0x7f3898104280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 shutdown_connections 2024-04-06T13:59:53.423 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T13:59:53.422+0000 7f389dc49700 1 -- 172.21.15.134:0/742882318 wait complete. 2024-04-06T13:59:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:53 smithi134 ceph-mon[47549]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:53.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:53 smithi105 ceph-mon[46969]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:53.757 DEBUG:teuthology.orchestra.run.smithi105:mgr.y> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.y.service 2024-04-06T13:59:53.760 DEBUG:teuthology.orchestra.run.smithi134:mgr.z> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.z.service 2024-04-06T13:59:53.763 INFO:tasks.cephadm:Deploying OSDs... 2024-04-06T13:59:53.763 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T13:59:53.763 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/scratch_devs of=/dev/stdout 2024-04-06T13:59:53.785 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-06T13:59:53.785 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_1 2024-04-06T13:59:53.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:53 smithi090 ceph-mon[43420]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:53.792 INFO:journalctl@ceph.mgr.y.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T13:59:53.793 INFO:journalctl@ceph.mgr.z.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T13:59:53.807 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 6h/6d Inode: 181315 Links: 1 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-06 13:59:13.602886653 +0000 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-06 13:52:34.035728411 +0000 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-06 13:52:34.035728411 +0000 2024-04-06T13:59:53.808 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: - 2024-04-06T13:59:53.809 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-06T13:59:53.877 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-06T13:59:53.877 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-06T13:59:53.877 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000290611 s, 1.8 MB/s 2024-04-06T13:59:53.879 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-06T13:59:53.939 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_2 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 6h/6d Inode: 183419 Links: 1 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-06 13:59:13.656885789 +0000 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-06 13:52:34.383722838 +0000 2024-04-06T13:59:53.998 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-06 13:52:34.383722838 +0000 2024-04-06T13:59:53.999 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: - 2024-04-06T13:59:53.999 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-06T13:59:54.066 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-06T13:59:54.066 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-06T13:59:54.066 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000219491 s, 2.3 MB/s 2024-04-06T13:59:54.067 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-06T13:59:54.126 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_3 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 6h/6d Inode: 181450 Links: 1 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-06 13:59:13.714884861 +0000 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-06 13:52:34.725717363 +0000 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-06 13:52:34.725717363 +0000 2024-04-06T13:59:54.185 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: - 2024-04-06T13:59:54.186 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-06T13:59:54.251 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-06T13:59:54.252 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-06T13:59:54.252 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000164172 s, 3.1 MB/s 2024-04-06T13:59:54.253 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-06T13:59:54.310 DEBUG:teuthology.orchestra.run.smithi090:> stat /dev/vg_nvme/lv_4 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Device: 6h/6d Inode: 179484 Links: 1 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Access: 2024-04-06 13:59:13.774883901 +0000 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Modify: 2024-04-06 13:52:35.070711842 +0000 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout:Change: 2024-04-06 13:52:35.070711842 +0000 2024-04-06T13:59:54.368 INFO:teuthology.orchestra.run.smithi090.stdout: Birth: - 2024-04-06T13:59:54.369 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-06T13:59:54.435 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records in 2024-04-06T13:59:54.435 INFO:teuthology.orchestra.run.smithi090.stderr:1+0 records out 2024-04-06T13:59:54.435 INFO:teuthology.orchestra.run.smithi090.stderr:512 bytes copied, 0.000227758 s, 2.2 MB/s 2024-04-06T13:59:54.436 DEBUG:teuthology.orchestra.run.smithi090:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-06T13:59:54.494 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-04-06T13:59:54.494 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-04-06T13:59:54.515 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-06T13:59:54.515 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-04-06T13:59:54.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi090=x;smithi105=y;smithi134=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:54.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: Saving service mgr spec with placement smithi090=x;smithi105=y;smithi134=z;count:3 2024-04-06T13:59:54.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: Deploying daemon mgr.z on smithi134 2024-04-06T13:59:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:54 smithi090 ceph-mon[43420]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 6h/6d Inode: 176998 Links: 1 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-04-06 13:59:42.265714623 +0000 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-04-06 13:52:36.397520488 +0000 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-04-06 13:52:36.397520488 +0000 2024-04-06T13:59:54.576 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: - 2024-04-06T13:59:54.577 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi090=x;smithi105=y;smithi134=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: Saving service mgr spec with placement smithi090=x;smithi105=y;smithi134=z;count:3 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.583 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:54.584 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:54.584 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:54.584 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.584 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: Deploying daemon mgr.z on smithi134 2024-04-06T13:59:54.584 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:54 smithi134 ceph-mon[47549]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:54.646 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-04-06T13:59:54.646 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-04-06T13:59:54.646 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000238042 s, 2.2 MB/s 2024-04-06T13:59:54.647 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-06T13:59:54.707 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi090=x;smithi105=y;smithi134=z", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: Saving service mgr spec with placement smithi090=x;smithi105=y;smithi134=z;count:3 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:54.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:54.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: Deploying daemon mgr.z on smithi134 2024-04-06T13:59:54.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:54 smithi105 ceph-mon[46969]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 6h/6d Inode: 181468 Links: 1 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-04-06 13:59:42.265714623 +0000 2024-04-06T13:59:54.766 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-04-06 13:52:36.750513403 +0000 2024-04-06T13:59:54.767 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-04-06 13:52:36.750513403 +0000 2024-04-06T13:59:54.767 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: - 2024-04-06T13:59:54.767 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-06T13:59:54.834 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-04-06T13:59:54.834 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-04-06T13:59:54.834 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000206279 s, 2.5 MB/s 2024-04-06T13:59:54.835 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-06T13:59:54.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:54 smithi134 systemd[1]: Starting Ceph mgr.z for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:59:54.894 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-04-06T13:59:54.953 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-06T13:59:54.953 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 6h/6d Inode: 177816 Links: 1 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-04-06 13:59:42.265714623 +0000 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-04-06 13:52:37.130505776 +0000 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-04-06 13:52:37.130505776 +0000 2024-04-06T13:59:54.954 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: - 2024-04-06T13:59:54.954 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-06T13:59:55.021 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-04-06T13:59:55.021 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-04-06T13:59:55.021 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000211746 s, 2.4 MB/s 2024-04-06T13:59:55.023 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-06T13:59:55.082 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-04-06T13:59:55.140 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-06T13:59:55.140 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:55.140 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 6h/6d Inode: 177132 Links: 1 2024-04-06T13:59:55.140 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:55.140 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:55.141 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-04-06 13:59:42.266714603 +0000 2024-04-06T13:59:55.141 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-04-06 13:52:37.483498690 +0000 2024-04-06T13:59:55.141 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-04-06 13:52:37.483498690 +0000 2024-04-06T13:59:55.141 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: - 2024-04-06T13:59:55.141 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-06T13:59:55.207 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-04-06T13:59:55.207 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-04-06T13:59:55.208 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000208457 s, 2.5 MB/s 2024-04-06T13:59:55.209 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-06T13:59:55.267 DEBUG:teuthology.orchestra.run.smithi134:> set -ex 2024-04-06T13:59:55.267 DEBUG:teuthology.orchestra.run.smithi134:> dd if=/scratch_devs of=/dev/stdout 2024-04-06T13:59:55.285 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-06T13:59:55.285 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_1 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 6h/6d Inode: 179445 Links: 1 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-04-06 13:59:54.631298818 +0000 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-04-06 13:52:35.113138776 +0000 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-04-06 13:52:35.113138776 +0000 2024-04-06T13:59:55.345 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-04-06T13:59:55.345 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-06T13:59:55.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 ceph-mgr[49433]: -- 172.21.15.134:0/1960966469 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5638760f81a0 con 0x5638760cf000 2024-04-06T13:59:55.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 bash[49395]: d5bab1fede1972f0784c0643d794863e93170a7c772a51b3f5cbbd62aca49bb1 2024-04-06T13:59:55.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 systemd[1]: Started Ceph mgr.z for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:59:55.416 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-04-06T13:59:55.416 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-04-06T13:59:55.416 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000224405 s, 2.3 MB/s 2024-04-06T13:59:55.417 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-06T13:59:55.475 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_2 2024-04-06T13:59:55.519 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:55.423+0000 7fb521866200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T13:59:55.533 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-06T13:59:55.533 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:55.533 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 6h/6d Inode: 178838 Links: 1 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-04-06 13:59:54.631298818 +0000 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-04-06 13:52:35.473130359 +0000 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-04-06 13:52:35.473130359 +0000 2024-04-06T13:59:55.534 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-04-06T13:59:55.534 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-06T13:59:55.601 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-04-06T13:59:55.601 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-04-06T13:59:55.601 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000220632 s, 2.3 MB/s 2024-04-06T13:59:55.602 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-06T13:59:55.660 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_3 2024-04-06T13:59:55.698 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:55.519+0000 7fb521866200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T13:59:55.699 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:55.612+0000 7fb521866200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T13:59:55.699 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:55.698+0000 7fb521866200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T13:59:55.715 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-06T13:59:55.715 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:55.715 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 6h/6d Inode: 181357 Links: 1 2024-04-06T13:59:55.715 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:55.715 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:55.716 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-04-06 13:59:54.631298818 +0000 2024-04-06T13:59:55.716 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-04-06 13:52:35.810122479 +0000 2024-04-06T13:59:55.716 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-04-06 13:52:35.810122479 +0000 2024-04-06T13:59:55.716 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-04-06T13:59:55.716 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-06T13:59:55.781 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-04-06T13:59:55.781 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-04-06T13:59:55.781 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.00019123 s, 2.7 MB/s 2024-04-06T13:59:55.782 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-06T13:59:55.840 DEBUG:teuthology.orchestra.run.smithi134:> stat /dev/vg_nvme/lv_4 2024-04-06T13:59:55.897 INFO:teuthology.orchestra.run.smithi134.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-06T13:59:55.897 INFO:teuthology.orchestra.run.smithi134.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-06T13:59:55.897 INFO:teuthology.orchestra.run.smithi134.stdout:Device: 6h/6d Inode: 180732 Links: 1 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout:Context: system_u:object_r:device_t:s0 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout:Access: 2024-04-06 13:59:54.631298818 +0000 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout:Modify: 2024-04-06 13:52:36.171114038 +0000 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout:Change: 2024-04-06 13:52:36.171114038 +0000 2024-04-06T13:59:55.898 INFO:teuthology.orchestra.run.smithi134.stdout: Birth: - 2024-04-06T13:59:55.898 DEBUG:teuthology.orchestra.run.smithi134:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-06T13:59:55.963 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records in 2024-04-06T13:59:55.963 INFO:teuthology.orchestra.run.smithi134.stderr:1+0 records out 2024-04-06T13:59:55.963 INFO:teuthology.orchestra.run.smithi134.stderr:512 bytes copied, 0.000216433 s, 2.4 MB/s 2024-04-06T13:59:55.964 DEBUG:teuthology.orchestra.run.smithi134:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-06T13:59:56.022 INFO:tasks.cephadm:Deploying osd.0 on smithi090 with /dev/vg_nvme/lv_4... 2024-04-06T13:59:56.022 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-06T13:59:56.113 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:56.114 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-mon[47549]: Deploying daemon mgr.y on smithi105 2024-04-06T13:59:56.249 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:56.355 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:56.356 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:56 smithi090 ceph-mon[43420]: Deploying daemon mgr.y on smithi105 2024-04-06T13:59:56.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:56.350+0000 7fb521866200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:59:56.497 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:56 smithi105 systemd[1]: Starting Ceph mgr.y for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:56.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mon[46969]: Deploying daemon mgr.y on smithi105 2024-04-06T13:59:56.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:56.601+0000 7fb521866200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T13:59:56.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:56.685+0000 7fb521866200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T13:59:56.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:56.794+0000 7fb521866200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T13:59:56.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:56 smithi105 ceph-mgr[48476]: -- 172.21.15.105:0/1384020239 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b5649ea1a0 con 0x55b5649c0c00 2024-04-06T13:59:56.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:56 smithi105 bash[48439]: 3b87c0798ca97e2148ec15d4de8ac376addb21bcd007aa2b48e47270523bc87a 2024-04-06T13:59:56.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:56 smithi105 systemd[1]: Started Ceph mgr.y for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T13:59:57.120 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:57.117+0000 7f06b9cf8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T13:59:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:56.954+0000 7fb521866200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:57.089+0000 7fb521866200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T13:59:57.394 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:57.180+0000 7fb521866200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:57.222+0000 7f06b9cf8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:57.320+0000 7f06b9cf8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:57.407+0000 7f06b9cf8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.408 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:57.409 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T13:59:57.548 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-06T13:59:57.548 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-06T13:59:57.549 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-06T13:59:57.549 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-06T13:59:57.549 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113657 s, 923 MB/s 2024-04-06T13:59:57.549 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-06T13:59:57.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:57.595+0000 7fb521866200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T13:59:57.897 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi090:vg_nvme/lv_4 2024-04-06T13:59:58.124 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T13:59:58.280 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:57.996+0000 7fb521866200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T13:59:58.281 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:58.188+0000 7fb521866200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T13:59:58.281 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:58.280+0000 7fb521866200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T13:59:58.393 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.061+0000 7f06b9cf8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T13:59:58.393 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.308+0000 7f06b9cf8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T13:59:58.393 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.393+0000 7f06b9cf8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T13:59:58.656 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.500+0000 7f06b9cf8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: Reconfiguring mgr.x (unknown last config time)... 2024-04-06T13:59:58.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:58.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:58.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:58.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: Reconfiguring daemon mgr.x on smithi090 2024-04-06T13:59:58.774 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:58 smithi090 ceph-mon[43420]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.656+0000 7f06b9cf8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.783+0000 7f06b9cf8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:58.867+0000 7f06b9cf8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: Reconfiguring mgr.x (unknown last config time)... 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: Reconfiguring daemon mgr.x on smithi090 2024-04-06T13:59:58.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:58 smithi105 ceph-mon[46969]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:59.044 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:59.044 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: Reconfiguring mgr.x (unknown last config time)... 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: Reconfiguring daemon mgr.x on smithi090 2024-04-06T13:59:59.045 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:58 smithi134 ceph-mon[47549]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T13:59:59.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:59.044+0000 7fb521866200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T13:59:59.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:59.143+0000 7fb521866200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T13:59:59.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:59.290+0000 7fb521866200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 -- 172.21.15.90:0/957126281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b60100f10 msgr2=0x7f3b60101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 --2- 172.21.15.90:0/957126281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b60100f10 0x7f3b60101370 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f3b5c0099c0 tx=0x7f3b5c009cd0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 -- 172.21.15.90:0/957126281 shutdown_connections 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 --2- 172.21.15.90:0/957126281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b601018b0 0x7f3b60108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 --2- 172.21.15.90:0/957126281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b60100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 --2- 172.21.15.90:0/957126281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b60100f10 0x7f3b60101370 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 -- 172.21.15.90:0/957126281 >> 172.21.15.90:0/957126281 conn(0x7f3b600fb2b0 msgr2=0x7f3b600fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 -- 172.21.15.90:0/957126281 shutdown_connections 2024-04-06T13:59:59.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 -- 172.21.15.90:0/957126281 wait complete. 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.402+0000 7f3b68044700 1 Processor -- start 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 -- start start 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b60100f10 0x7f3b6019ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 0x7f3b6019e080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:59.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b6019e820 con 0x7f3b601018b0 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b6019e990 con 0x7f3b600ffd10 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b68044700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b6019eb50 con 0x7f3b60100f10 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b665e1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 0x7f3b6019e080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:47242/0 (socket says 172.21.15.90:47242) 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 -- 172.21.15.90:0/864806648 learned_addr learned my addr 172.21.15.90:0/864806648 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b665e1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 0x7f3b6019e080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:33952/0 (socket says 172.21.15.90:33952) 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b655df700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b60100f10 0x7f3b6019ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 -- 172.21.15.90:0/864806648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b60100f10 msgr2=0x7f3b6019ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b60100f10 0x7f3b6019ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 -- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 msgr2=0x7f3b6019e080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 0x7f3b6019e080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T13:59:59.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 -- 172.21.15.90:0/864806648 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b5c0095f0 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.403+0000 7f3b65de0700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3b5400e970 tx=0x7f3b5400ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b54009790 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b6019ee30 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b5400c7e0 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b54003c70 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b6019f250 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.404+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 13) v1 ==== 45597+0+0 (secure 0 0 0) 0x7f3b54018430 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.405+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b60107190 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.405+0000 7f3b52ffd700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 0x7f3b4c03ae90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.405+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f3b54014070 con 0x7f3b600ffd10 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.405+0000 7f3b655df700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 0x7f3b4c03ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T13:59:59.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.405+0000 7f3b655df700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 0x7f3b4c03ae90 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f3b5c003730 tx=0x7f3b5c00b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T13:59:59.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.410+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3b54003dd0 con 0x7f3b600ffd10 2024-04-06T13:59:59.497 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:59.238+0000 7f06b9cf8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T13:59:59.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T13:59:59.609+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f3b6002cda0 con 0x7f3b4c0389d0 2024-04-06T13:59:59.905 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:59.622+0000 7f06b9cf8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T13:59:59.905 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:59.813+0000 7f06b9cf8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T13:59:59.905 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T13:59:59.904+0000 7f06b9cf8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:00.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:00.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 13:59:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:00.265 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 13:59:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.280 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T13:59:59.911+0000 7fb521866200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T14:00:00.280 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:00.091+0000 7fb521866200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:00.280+0000 7fb521866200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:00.281 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 13:59:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:00.984 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:00.660+0000 7f06b9cf8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T14:00:00.985 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:00.755+0000 7f06b9cf8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T14:00:00.985 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:00.902+0000 7f06b9cf8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T14:00:01.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-mon[46969]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:01.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-mon[46969]: overall HEALTH_OK 2024-04-06T14:00:01.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:00 smithi105 ceph-mon[46969]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:00 smithi090 ceph-mon[43420]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:00 smithi090 ceph-mon[43420]: overall HEALTH_OK 2024-04-06T14:00:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:00 smithi090 ceph-mon[43420]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:01.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:01.207+0000 7fb521866200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T14:00:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:00 smithi134 ceph-mon[47549]: from='client.24122 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:00 smithi134 ceph-mon[47549]: overall HEALTH_OK 2024-04-06T14:00:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:00 smithi134 ceph-mon[47549]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:01.766 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:01.490+0000 7fb521866200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:00:01.766 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:01.577+0000 7fb521866200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T14:00:01.766 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:00:01.766+0000 7fb521866200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T14:00:01.896 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:01.520+0000 7f06b9cf8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T14:00:01.896 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:01.701+0000 7f06b9cf8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T14:00:01.896 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:01.895+0000 7f06b9cf8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T14:00:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]: dispatch 2024-04-06T14:00:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]': finished 2024-04-06T14:00:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: osdmap e5: 1 total, 0 up, 1 in 2024-04-06T14:00:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: Standby manager daemon z started 2024-04-06T14:00:02.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:00:02.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:02.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:00:02.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:01 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]': finished 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: osdmap e5: 1 total, 0 up, 1 in 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: Standby manager daemon z started 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:00:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:01 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:02.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]: dispatch 2024-04-06T14:00:02.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/962954388' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3c2803b-07fd-4e23-b926-5e9dcc735ce7"}]': finished 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: osdmap e5: 1 total, 0 up, 1 in 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: Standby manager daemon z started 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:00:02.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:01 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/619166365' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:02.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:02.431+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mgrmap(e 14) v1 ==== 90707+0+0 (secure 0 0 0) 0x7f3b540214b0 con 0x7f3b600ffd10 2024-04-06T14:00:03.101 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:02.817+0000 7f06b9cf8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T14:00:03.101 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/75470616' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:03.101 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:02 smithi105 ceph-mon[46969]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:03.101 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:02 smithi105 ceph-mon[46969]: mgrmap e14: x(active, since 58s), standbys: z 2024-04-06T14:00:03.101 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:02 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:00:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/75470616' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:02 smithi090 ceph-mon[43420]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:02 smithi090 ceph-mon[43420]: mgrmap e14: x(active, since 58s), standbys: z 2024-04-06T14:00:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:02 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:00:03.377 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:03.100+0000 7f06b9cf8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:00:03.377 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:03.188+0000 7f06b9cf8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T14:00:03.377 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:00:03.376+0000 7f06b9cf8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T14:00:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/75470616' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:02 smithi134 ceph-mon[47549]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:02 smithi134 ceph-mon[47549]: mgrmap e14: x(active, since 58s), standbys: z 2024-04-06T14:00:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:02 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:00:04.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:04.001+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mon.2 v2:172.21.15.105:3300/0 8 ==== mgrmap(e 15) v1 ==== 135817+0+0 (secure 0 0 0) 0x7f3b5400cc50 con 0x7f3b600ffd10 2024-04-06T14:00:04.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-mon[46969]: Standby manager daemon y started 2024-04-06T14:00:04.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:00:04.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:04.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:00:04.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:03 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:03 smithi090 ceph-mon[43420]: Standby manager daemon y started 2024-04-06T14:00:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:03 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:00:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:03 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:04.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:03 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:00:04.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:03 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:03 smithi134 ceph-mon[47549]: Standby manager daemon y started 2024-04-06T14:00:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:03 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:00:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:03 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:00:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:03 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:00:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:03 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/220828432' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:00:05.281 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:05 smithi090 ceph-mon[43420]: mgrmap e15: x(active, since 59s), standbys: z, y 2024-04-06T14:00:05.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:00:05.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:05 smithi090 ceph-mon[43420]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:05 smithi134 ceph-mon[47549]: mgrmap e15: x(active, since 59s), standbys: z, y 2024-04-06T14:00:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:00:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:05 smithi134 ceph-mon[47549]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:05 smithi105 ceph-mon[46969]: mgrmap e15: x(active, since 59s), standbys: z, y 2024-04-06T14:00:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:00:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:05 smithi105 ceph-mon[46969]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:07.210 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:07 smithi090 ceph-mon[43420]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:07 smithi105 ceph-mon[46969]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:07 smithi134 ceph-mon[47549]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:08.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-06T14:00:08.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:08.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:08 smithi090 ceph-mon[43420]: Deploying daemon osd.0 on smithi090 2024-04-06T14:00:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-06T14:00:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:08 smithi105 ceph-mon[46969]: Deploying daemon osd.0 on smithi090 2024-04-06T14:00:08.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-06T14:00:08.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:08.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:08 smithi134 ceph-mon[47549]: Deploying daemon osd.0 on smithi090 2024-04-06T14:00:09.453 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:09 smithi090 ceph-mon[43420]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:09.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:09 smithi105 ceph-mon[46969]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:09.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:09 smithi134 ceph-mon[47549]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:11.442 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:11 smithi090 ceph-mon[43420]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:11.442 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.442 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.442 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:11 smithi105 ceph-mon[46969]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:11 smithi134 ceph-mon[47549]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:12.953 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 0 on host 'smithi090' 2024-04-06T14:00:12.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.952+0000 7f3b52ffd700 1 -- 172.21.15.90:0/864806648 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f3b6002cda0 con 0x7f3b4c0389d0 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 msgr2=0x7f3b4c03ae90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 0x7f3b4c03ae90 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f3b5c003730 tx=0x7f3b5c00b540 comp rx=0 tx=0).stop 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 msgr2=0x7f3b600692d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f3b5400e970 tx=0x7f3b5400ec80 comp rx=0 tx=0).stop 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 shutdown_connections 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3b4c0389d0 0x7f3b4c03ae90 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:12.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3b601018b0 0x7f3b6019e080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:12.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b600ffd10 0x7f3b600692d0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:12.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 --2- 172.21.15.90:0/864806648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3b60100f10 0x7f3b6019ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:12.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 >> 172.21.15.90:0/864806648 conn(0x7f3b600fb2b0 msgr2=0x7f3b601042d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:12.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 shutdown_connections 2024-04-06T14:00:12.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:12.954+0000 7f3b68044700 1 -- 172.21.15.90:0/864806648 wait complete. 2024-04-06T14:00:13.349 DEBUG:teuthology.orchestra.run.smithi090:osd.0> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.0.service 2024-04-06T14:00:13.351 INFO:tasks.cephadm:Deploying osd.1 on smithi105 with /dev/vg_nvme/lv_4... 2024-04-06T14:00:13.351 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-06T14:00:13.380 INFO:journalctl@ceph.osd.0.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:00:13.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:13 smithi090 ceph-mon[43420]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:13 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:13 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:13 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:13 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.581 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:00:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:13 smithi134 ceph-mon[47549]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:13 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:13 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:13 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:13 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.678 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:13 smithi105 ceph-mon[46969]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:13.678 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:13 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.678 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:13 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.679 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:13 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:13.679 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:13 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112402 s, 933 MB/s 2024-04-06T14:00:14.830 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-04-06T14:00:15.093 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:14 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:00:14.827+0000 7fba77ac0640 -1 osd.0 0 log_to_monitors true 2024-04-06T14:00:15.206 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi105:vg_nvme/lv_4 2024-04-06T14:00:15.421 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:00:15.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:15 smithi134 ceph-mon[47549]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:15.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:15 smithi134 ceph-mon[47549]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:15.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:15 smithi134 ceph-mon[47549]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:15.691 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:15 smithi105 ceph-mon[46969]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:15.692 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:15 smithi105 ceph-mon[46969]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:15.692 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:15 smithi105 ceph-mon[46969]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:15.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:15 smithi090 ceph-mon[43420]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:15.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:15 smithi090 ceph-mon[43420]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:15.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:15 smithi090 ceph-mon[43420]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-06T14:00:16.365 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 -- 172.21.15.105:0/3612378056 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0038100f10 msgr2=0x7f0038101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:16.365 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 --2- 172.21.15.105:0/3612378056 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0038100f10 0x7f0038101370 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f00340099c0 tx=0x7f0034009cd0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.365 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 -- 172.21.15.105:0/3612378056 shutdown_connections 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 --2- 172.21.15.105:0/3612378056 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00381018b0 0x7f0038108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 --2- 172.21.15.105:0/3612378056 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00380ffd10 0x7f0038100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 --2- 172.21.15.105:0/3612378056 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0038100f10 0x7f0038101370 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.362+0000 7f003f333700 1 -- 172.21.15.105:0/3612378056 >> 172.21.15.105:0/3612378056 conn(0x7f00380fb2b0 msgr2=0x7f00380fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- 172.21.15.105:0/3612378056 shutdown_connections 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- 172.21.15.105:0/3612378056 wait complete. 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 Processor -- start 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- start start 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00380ffd10 0x7f0038196d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0038100f10 0x7f00381972d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:16.366 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f003819e8d0 con 0x7f00380ffd10 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f003819ea10 con 0x7f00381018b0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003f333700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f003819eba0 con 0x7f0038100f10 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003d8d0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003d8d0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:48084/0 (socket says 172.21.15.105:48084) 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003d8d0700 1 -- 172.21.15.105:0/2921685551 learned_addr learned my addr 172.21.15.105:0/2921685551 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.363+0000 7f003c8ce700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0038100f10 0x7f00381972d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d0cf700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00380ffd10 0x7f0038196d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 -- 172.21.15.105:0/2921685551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0038100f10 msgr2=0x7f00381972d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0038100f10 0x7f00381972d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 -- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00380ffd10 msgr2=0x7f0038196d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00380ffd10 0x7f0038196d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 -- 172.21.15.105:0/2921685551 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00340095f0 con 0x7f00381018b0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003d8d0700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f0030007ee0 tx=0x7f003000d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:16.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f003000dd20 con 0x7f00381018b0 2024-04-06T14:00:16.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f003000f040 con 0x7f00381018b0 2024-04-06T14:00:16.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f003819ee80 con 0x7f00381018b0 2024-04-06T14:00:16.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00300048d0 con 0x7f00381018b0 2024-04-06T14:00:16.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.364+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f003819f2f0 con 0x7f00381018b0 2024-04-06T14:00:16.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.365+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f003804ea80 con 0x7f00381018b0 2024-04-06T14:00:16.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.366+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 15) v1 ==== 135817+0+0 (secure 0 0 0) 0x7f0030003a00 con 0x7f00381018b0 2024-04-06T14:00:16.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.366+0000 7f002e7fc700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 0x7f00240a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:16.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.367+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1361+0+0 (secure 0 0 0) 0x7f00300ca4f0 con 0x7f00381018b0 2024-04-06T14:00:16.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.367+0000 7f003d0cf700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 0x7f00240a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:16.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.367+0000 7f003d0cf700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 0x7f00240a3650 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f0038100d70 tx=0x7f002800b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:16.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.371+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f00300957e0 con 0x7f00381018b0 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: osdmap e6: 1 total, 0 up, 1 in 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi090 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 17113M 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:16 smithi105 ceph-mon[46969]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:16.574 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:16.573+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f00380fd500 con 0x7f00240a1190 2024-04-06T14:00:16.788 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:00:16.396+0000 7fba6d124700 -1 osd.0 0 waiting for initial osdmap 2024-04-06T14:00:16.788 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:00:16.399+0000 7fba68f19700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: osdmap e6: 1 total, 0 up, 1 in 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi090 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 17113M 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:16 smithi090 ceph-mon[43420]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:16.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-06T14:00:16.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: osdmap e6: 1 total, 0 up, 1 in 2024-04-06T14:00:16.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi090 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 17113M 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:16.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:16.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:16.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:16.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:16 smithi134 ceph-mon[47549]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: osdmap e7: 1 total, 0 up, 1 in 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:17.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:00:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: osdmap e7: 1 total, 0 up, 1 in 2024-04-06T14:00:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:17.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: osdmap e7: 1 total, 0 up, 1 in 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:17.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:00:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:00:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] boot 2024-04-06T14:00:18.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: osdmap e8: 1 total, 1 up, 1 in 2024-04-06T14:00:18.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:18.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:18 smithi134 ceph-mon[47549]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] boot 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: osdmap e8: 1 total, 1 up, 1 in 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:18 smithi105 ceph-mon[46969]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:18.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:00:18.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:00:18.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: osd.0 [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] boot 2024-04-06T14:00:18.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: osdmap e8: 1 total, 1 up, 1 in 2024-04-06T14:00:18.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:00:18.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:18 smithi090 ceph-mon[43420]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/867971726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: osdmap e9: 1 total, 1 up, 1 in 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]': finished 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: osdmap e10: 2 total, 1 up, 2 in 2024-04-06T14:00:19.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:19.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:19 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/3701892218' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:19.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/867971726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: osdmap e9: 1 total, 1 up, 1 in 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]': finished 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: osdmap e10: 2 total, 1 up, 2 in 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:19.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:19 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/3701892218' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:19.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/867971726' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]: dispatch 2024-04-06T14:00:19.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: osdmap e9: 1 total, 1 up, 1 in 2024-04-06T14:00:19.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "efe81cdf-a49b-44a5-aece-bef13902a718"}]': finished 2024-04-06T14:00:19.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: osdmap e10: 2 total, 1 up, 2 in 2024-04-06T14:00:19.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:19.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:19 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/3701892218' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:20.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:20 smithi105 ceph-mon[46969]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:20.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:20 smithi090 ceph-mon[43420]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:20.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:20 smithi134 ceph-mon[47549]: pgmap v38: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:23.477 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:23 smithi105 ceph-mon[46969]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:23.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:23 smithi090 ceph-mon[43420]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:23 smithi134 ceph-mon[47549]: pgmap v39: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:25.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:25 smithi105 ceph-mon[46969]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:25.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-06T14:00:25.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:25.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:25 smithi105 ceph-mon[46969]: Deploying daemon osd.1 on smithi105 2024-04-06T14:00:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:25 smithi090 ceph-mon[43420]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-06T14:00:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:25.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:25 smithi090 ceph-mon[43420]: Deploying daemon osd.1 on smithi105 2024-04-06T14:00:25.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:25 smithi134 ceph-mon[47549]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:25.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-06T14:00:25.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:25.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:25 smithi134 ceph-mon[47549]: Deploying daemon osd.1 on smithi105 2024-04-06T14:00:27.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:27 smithi105 ceph-mon[46969]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:27 smithi090 ceph-mon[43420]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:27 smithi134 ceph-mon[47549]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:28 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:28 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:28 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:28 smithi134 ceph-mon[47549]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:29.161 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:28 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.161 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:28 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.161 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:28 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:29.161 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:28 smithi105 ceph-mon[46969]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:28 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:28 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:29.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:28 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:29.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:28 smithi090 ceph-mon[43420]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:30.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.078+0000 7f002e7fc700 1 -- 172.21.15.105:0/2921685551 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f00380fd500 con 0x7f00240a1190 2024-04-06T14:00:30.079 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 1 on host 'smithi105' 2024-04-06T14:00:30.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 msgr2=0x7f00240a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 0x7f00240a3650 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f0038100d70 tx=0x7f002800b410 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 msgr2=0x7f003819e300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f0030007ee0 tx=0x7f003000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 shutdown_connections 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00240a1190 0x7f00240a3650 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00381018b0 0x7f003819e300 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00380ffd10 0x7f0038196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 --2- 172.21.15.105:0/2921685551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0038100f10 0x7f00381972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 >> 172.21.15.105:0/2921685551 conn(0x7f00380fb2b0 msgr2=0x7f00380fcdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 shutdown_connections 2024-04-06T14:00:30.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:00:30.080+0000 7f003f333700 1 -- 172.21.15.105:0/2921685551 wait complete. 2024-04-06T14:00:30.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:30 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:30 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:30.450 DEBUG:teuthology.orchestra.run.smithi105:osd.1> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.1.service 2024-04-06T14:00:30.452 INFO:tasks.cephadm:Deploying osd.2 on smithi134 with /dev/vg_nvme/lv_4... 2024-04-06T14:00:30.453 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-06T14:00:30.485 INFO:journalctl@ceph.osd.1.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:00:30.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:30.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:30.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:30.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:30 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:30.680 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:00:31.243 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:31 smithi105 ceph-mon[46969]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:31.378 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:31 smithi134 ceph-mon[47549]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:31 smithi090 ceph-mon[43420]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:31.895 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-06T14:00:31.896 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-06T14:00:31.896 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-04-06T14:00:31.896 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-04-06T14:00:31.896 INFO:teuthology.orchestra.run.smithi134.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0111339 s, 942 MB/s 2024-04-06T14:00:31.896 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-04-06T14:00:32.116 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:00:31.827+0000 7fb35501e640 -1 osd.1 0 log_to_monitors true 2024-04-06T14:00:32.254 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:32 smithi134 ceph-mon[47549]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:32.254 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:32 smithi134 ceph-mon[47549]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:32.255 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi134:vg_nvme/lv_4 2024-04-06T14:00:32.473 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:00:32.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:32 smithi105 ceph-mon[46969]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:32.478 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:32 smithi105 ceph-mon[46969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:32.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:32 smithi090 ceph-mon[43420]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:32.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:32 smithi090 ceph-mon[43420]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 -- 172.21.15.134:0/719933320 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 msgr2=0x7f411c108f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 --2- 172.21.15.134:0/719933320 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c108f70 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f411400b0a0 tx=0x7f411400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 -- 172.21.15.134:0/719933320 shutdown_connections 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 --2- 172.21.15.134:0/719933320 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c108f70 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 --2- 172.21.15.134:0/719933320 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 0x7f411c103c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 --2- 172.21.15.134:0/719933320 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 0x7f411c1065c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 -- 172.21.15.134:0/719933320 >> 172.21.15.134:0/719933320 conn(0x7f411c0fb2b0 msgr2=0x7f411c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 -- 172.21.15.134:0/719933320 shutdown_connections 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.388+0000 7f41228e6700 1 -- 172.21.15.134:0/719933320 wait complete. 2024-04-06T14:00:33.391 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 Processor -- start 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 -- start start 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 0x7f411c196e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 0x7f411c197340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411c19e940 con 0x7f411c1018a0 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411c19ea80 con 0x7f411c1041d0 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f41228e6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f411c19ec10 con 0x7f411c106b00 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:53818/0 (socket says 172.21.15.134:53818) 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 -- 172.21.15.134:0/577618708 learned_addr learned my addr 172.21.15.134:0/577618708 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f411b7fe700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 0x7f411c197340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f411bfff700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 0x7f411c196e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 -- 172.21.15.134:0/577618708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 msgr2=0x7f411c197340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:33.392 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 0x7f411c197340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 -- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 msgr2=0x7f411c196e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 0x7f411c196e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 -- 172.21.15.134:0/577618708 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4114009d00 con 0x7f411c106b00 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.389+0000 7f4120e83700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4114007960 tx=0x7f411400bb60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.390+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f411400e040 con 0x7f411c106b00 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.390+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f411c19ee90 con 0x7f411c106b00 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.390+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4114004e60 con 0x7f411c106b00 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.390+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4114005330 con 0x7f411c106b00 2024-04-06T14:00:33.393 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.390+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f411c19f2a0 con 0x7f411c106b00 2024-04-06T14:00:33.394 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.391+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f411c0fce30 con 0x7f411c106b00 2024-04-06T14:00:33.394 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.392+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 15) v1 ==== 135817+0+0 (secure 0 0 0) 0x7f411401d9c0 con 0x7f411c106b00 2024-04-06T14:00:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-06T14:00:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: osdmap e11: 2 total, 1 up, 2 in 2024-04-06T14:00:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:33.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.396 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.392+0000 7f41197fa700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 0x7f41040a3550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:33.396 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.393+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(12..12 src has 1..12) v4 ==== 2029+0+0 (secure 0 0 0) 0x7f41140cc5a0 con 0x7f411c106b00 2024-04-06T14:00:33.396 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.393+0000 7f411bfff700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 0x7f41040a3550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:33.396 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.393+0000 7f411bfff700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 0x7f41040a3550 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f410c009bf0 tx=0x7f410c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:33.397 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.397+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4114097690 con 0x7f411c106b00 2024-04-06T14:00:33.497 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:00:33.138+0000 7fb34a682700 -1 osd.1 0 waiting for initial osdmap 2024-04-06T14:00:33.497 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:00:33.142+0000 7fb346477700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:00:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: osdmap e11: 2 total, 1 up, 2 in 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: osdmap e11: 2 total, 1 up, 2 in 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:33.594 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:33.593+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f411c02cd70 con 0x7f41040a1090 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi105 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 17114M 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: osdmap e12: 2 total, 1 up, 2 in 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.329 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.330 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:34.330 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:34.330 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:34 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi105 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 17114M 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: osdmap e12: 2 total, 1 up, 2 in 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:34.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:34 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi105 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 17114M 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: osdmap e12: 2 total, 1 up, 2 in 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:34 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] boot 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: osdmap e13: 2 total, 2 up, 2 in 2024-04-06T14:00:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] boot 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: osdmap e13: 2 total, 2 up, 2 in 2024-04-06T14:00:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:00:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: osd.1 [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] boot 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: osdmap e13: 2 total, 2 up, 2 in 2024-04-06T14:00:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: osdmap e14: 2 total, 2 up, 2 in 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/2630781699' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]': finished 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: osdmap e15: 3 total, 2 up, 3 in 2024-04-06T14:00:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:36.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/1551375290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: osdmap e14: 2 total, 2 up, 2 in 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/2630781699' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]': finished 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: osdmap e15: 3 total, 2 up, 3 in 2024-04-06T14:00:36.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:36.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/1551375290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: osdmap e14: 2 total, 2 up, 2 in 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/2630781699' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]: dispatch 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb0a1674-931b-4a33-80b7-05b28a2b8f86"}]': finished 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: osdmap e15: 3 total, 2 up, 3 in 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/1551375290' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:37 smithi134 ceph-mon[47549]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:37.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:37 smithi105 ceph-mon[46969]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:37.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:37 smithi090 ceph-mon[43420]: pgmap v51: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:38 smithi105 ceph-mon[46969]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:38.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:38 smithi090 ceph-mon[43420]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:38 smithi134 ceph-mon[47549]: pgmap v52: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:41 smithi105 ceph-mon[46969]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-06T14:00:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:41.515 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:41 smithi134 ceph-mon[47549]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:41.515 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-06T14:00:41.515 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:41 smithi090 ceph-mon[43420]: pgmap v53: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-06T14:00:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:42.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:42 smithi105 ceph-mon[46969]: Deploying daemon osd.2 on smithi134 2024-04-06T14:00:42.504 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:42 smithi134 ceph-mon[47549]: Deploying daemon osd.2 on smithi134 2024-04-06T14:00:42.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:42 smithi090 ceph-mon[43420]: Deploying daemon osd.2 on smithi134 2024-04-06T14:00:43.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:43 smithi134 ceph-mon[47549]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:43.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:43 smithi105 ceph-mon[46969]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:43 smithi090 ceph-mon[43420]: pgmap v54: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:45 smithi134 ceph-mon[47549]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:45.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:45.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:45.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:45 smithi105 ceph-mon[46969]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:45 smithi090 ceph-mon[43420]: pgmap v55: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.695 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 2 on host 'smithi134' 2024-04-06T14:00:46.695 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.695+0000 7f41197fa700 1 -- 172.21.15.134:0/577618708 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f411c02cd70 con 0x7f41040a1090 2024-04-06T14:00:46.698 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 msgr2=0x7f41040a3550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:46.698 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 0x7f41040a3550 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f410c009bf0 tx=0x7f410c009260 comp rx=0 tx=0).stop 2024-04-06T14:00:46.698 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 msgr2=0x7f411c19e370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:46.698 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4114007960 tx=0x7f411400bb60 comp rx=0 tx=0).stop 2024-04-06T14:00:46.698 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 shutdown_connections 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f41040a1090 0x7f41040a3550 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f411c106b00 0x7f411c19e370 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f411c1018a0 0x7f411c196e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 --2- 172.21.15.134:0/577618708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f411c1041d0 0x7f411c197340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 >> 172.21.15.134:0/577618708 conn(0x7f411c0fb2b0 msgr2=0x7f411c102160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 shutdown_connections 2024-04-06T14:00:46.699 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:00:46.698+0000 7f41228e6700 1 -- 172.21.15.134:0/577618708 wait complete. 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.841 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:46 smithi134 ceph-mon[47549]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:46 smithi105 ceph-mon[46969]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:47.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:47.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:46 smithi090 ceph-mon[43420]: pgmap v56: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:47.136 DEBUG:teuthology.orchestra.run.smithi134:osd.2> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.2.service 2024-04-06T14:00:47.138 INFO:tasks.cephadm:Deploying osd.3 on smithi090 with /dev/vg_nvme/lv_3... 2024-04-06T14:00:47.139 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-06T14:00:47.168 INFO:journalctl@ceph.osd.2.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:00:47.373 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:00:47.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:47.808 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.809 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.809 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:47.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:47.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0117397 s, 893 MB/s 2024-04-06T14:00:48.721 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-06T14:00:48.893 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:00:48.627+0000 7fd9f47e9640 -1 osd.2 0 log_to_monitors true 2024-04-06T14:00:48.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:48 smithi134 ceph-mon[47549]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:48.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:48 smithi134 ceph-mon[47549]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:48.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:48 smithi134 ceph-mon[47549]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:48.895 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:48 smithi090 ceph-mon[43420]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:48.895 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:48 smithi090 ceph-mon[43420]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:48.896 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:48 smithi090 ceph-mon[43420]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:48.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:48 smithi105 ceph-mon[46969]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:48.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:48 smithi105 ceph-mon[46969]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:48.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:48 smithi105 ceph-mon[46969]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-06T14:00:49.100 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi090:vg_nvme/lv_3 2024-04-06T14:00:49.326 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: osdmap e16: 3 total, 2 up, 3 in 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi134 2024-04-06T14:00:49.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 17114M 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:49.780 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-06T14:00:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: osdmap e16: 3 total, 2 up, 3 in 2024-04-06T14:00:49.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi134 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 17114M 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:49.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:50.143 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:00:49.656+0000 7fd9e9e4d700 -1 osd.2 0 waiting for initial osdmap 2024-04-06T14:00:50.143 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:00:49.660+0000 7fd9e5c42700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:00:50.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-06T14:00:50.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: osdmap e16: 3 total, 2 up, 3 in 2024-04-06T14:00:50.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi134 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 17114M 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:00:50.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 -- 172.21.15.90:0/3315063487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc0ffd10 msgr2=0x7f42bc100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 --2- 172.21.15.90:0/3315063487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc0ffd10 0x7f42bc100130 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f42ac0099c0 tx=0x7f42ac009cd0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 -- 172.21.15.90:0/3315063487 shutdown_connections 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 --2- 172.21.15.90:0/3315063487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc1018b0 0x7f42bc108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 --2- 172.21.15.90:0/3315063487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc0ffd10 0x7f42bc100130 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 --2- 172.21.15.90:0/3315063487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 -- 172.21.15.90:0/3315063487 >> 172.21.15.90:0/3315063487 conn(0x7f42bc0fb2b0 msgr2=0x7f42bc0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 -- 172.21.15.90:0/3315063487 shutdown_connections 2024-04-06T14:00:50.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.364+0000 7f42c1027700 1 -- 172.21.15.90:0/3315063487 wait complete. 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 Processor -- start 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 -- start start 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc0ffd10 0x7f42bc103c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc1022f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc102f10 con 0x7f42bc1018b0 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc103080 con 0x7f42bc100f10 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42c1027700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42bc1a0e50 con 0x7f42bc0ffd10 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42bb59e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42bb59e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:32796/0 (socket says 172.21.15.90:32796) 2024-04-06T14:00:50.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.365+0000 7f42bb59e700 1 -- 172.21.15.90:0/3989048616 learned_addr learned my addr 172.21.15.90:0/3989048616 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42ba59c700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc1022f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bad9d700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc0ffd10 0x7f42bc103c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 -- 172.21.15.90:0/3989048616 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc0ffd10 msgr2=0x7f42bc103c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc0ffd10 0x7f42bc103c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 -- 172.21.15.90:0/3989048616 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 msgr2=0x7f42bc1022f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc1022f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 -- 172.21.15.90:0/3989048616 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42ac0095f0 con 0x7f42bc1018b0 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42ba59c700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc1022f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42bb59e700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f42b400b970 tx=0x7f42b400bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:50.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42b400c930 con 0x7f42bc1018b0 2024-04-06T14:00:50.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42bc1a10b0 con 0x7f42bc1018b0 2024-04-06T14:00:50.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f42b4014440 con 0x7f42bc1018b0 2024-04-06T14:00:50.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42b40075d0 con 0x7f42bc1018b0 2024-04-06T14:00:50.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.366+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42bc1a15d0 con 0x7f42bc1018b0 2024-04-06T14:00:50.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.367+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42bc04ea80 con 0x7f42bc1018b0 2024-04-06T14:00:50.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.368+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 15) v1 ==== 135817+0+0 (secure 0 0 0) 0x7f42b40145b0 con 0x7f42bc1018b0 2024-04-06T14:00:50.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.368+0000 7f42a3fff700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 0x7f42a40a35d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:00:50.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.369+0000 7f42bad9d700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 0x7f42a40a35d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:00:50.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.369+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2571+0+0 (secure 0 0 0) 0x7f42b4006e40 con 0x7f42bc1018b0 2024-04-06T14:00:50.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.369+0000 7f42bad9d700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 0x7f42a40a35d0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f42ac003820 tx=0x7f42ac005ec0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:00:50.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.373+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f42b4095650 con 0x7f42bc1018b0 2024-04-06T14:00:50.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:50.575+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f42bc068e70 con 0x7f42a40a1110 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: osdmap e17: 3 total, 2 up, 3 in 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:50.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:50.994 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:50.994 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: osdmap e17: 3 total, 2 up, 3 in 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:00:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: osdmap e17: 3 total, 2 up, 3 in 2024-04-06T14:00:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-06T14:00:51.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:00:51.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:00:51.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:00:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:00:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:51.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] boot 2024-04-06T14:00:51.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: osdmap e18: 3 total, 3 up, 3 in 2024-04-06T14:00:51.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] boot 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: osdmap e18: 3 total, 3 up, 3 in 2024-04-06T14:00:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: osd.2 [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] boot 2024-04-06T14:00:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: osdmap e18: 3 total, 3 up, 3 in 2024-04-06T14:00:52.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: osdmap e19: 3 total, 3 up, 3 in 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: pgmap v63: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]: dispatch 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]': finished 2024-04-06T14:00:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: osdmap e20: 4 total, 3 up, 4 in 2024-04-06T14:00:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: osdmap e19: 3 total, 3 up, 3 in 2024-04-06T14:00:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: pgmap v63: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]: dispatch 2024-04-06T14:00:53.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:53.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]': finished 2024-04-06T14:00:53.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: osdmap e20: 4 total, 3 up, 4 in 2024-04-06T14:00:53.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:53.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: osdmap e19: 3 total, 3 up, 3 in 2024-04-06T14:00:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: pgmap v63: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]: dispatch 2024-04-06T14:00:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:53.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3918571510' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b2f41b24-c336-46cd-af3a-f3cf7d83e3cf"}]': finished 2024-04-06T14:00:53.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: osdmap e20: 4 total, 3 up, 4 in 2024-04-06T14:00:53.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:53.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:00:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:53 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/856206407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:53 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:53 smithi105 ceph-mon[46969]: osdmap e21: 4 total, 3 up, 4 in 2024-04-06T14:00:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:53 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:54.030 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:53 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/856206407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:54.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:53 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:54.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:53 smithi090 ceph-mon[43420]: osdmap e21: 4 total, 3 up, 4 in 2024-04-06T14:00:54.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:53 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:54.031 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:53 smithi090 sudo[68573]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:54.031 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:53 smithi090 sudo[68573]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:54.031 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:53 smithi090 sudo[68573]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:54.031 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:53 smithi090 sudo[68573]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:54.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:53 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/856206407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:00:54.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:53 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:00:54.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:53 smithi134 ceph-mon[47549]: osdmap e21: 4 total, 3 up, 4 in 2024-04-06T14:00:54.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:53 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:54.288 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:54 smithi090 sudo[68760]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:54.288 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:54 smithi090 sudo[68760]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:54.288 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:54 smithi090 sudo[68760]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:54.289 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:00:54 smithi090 sudo[68760]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57542]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57542]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57542]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57542]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57545]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57545]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:54.747 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57545]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:54.748 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:00:54 smithi105 sudo[57545]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:54 smithi134 sudo[58638]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:54 smithi134 sudo[58638]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:54 smithi134 sudo[58638]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:54 smithi134 sudo[58638]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58641]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58641]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:55.064 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58641]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:55.065 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58641]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:55.316 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.316 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 ceph-mon[43420]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:55.316 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 ceph-mon[43420]: osdmap e22: 4 total, 3 up, 4 in 2024-04-06T14:00:55.316 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:55.317 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 ceph-mon[47549]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 ceph-mon[47549]: osdmap e22: 4 total, 3 up, 4 in 2024-04-06T14:00:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:55 smithi105 ceph-mon[46969]: pgmap v66: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:55 smithi105 ceph-mon[46969]: osdmap e22: 4 total, 3 up, 4 in 2024-04-06T14:00:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:00:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:55.568 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69320]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:55.568 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69320]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:55.568 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69320]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:55.568 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69320]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:55.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:00:55.581+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f42b4095a20 con 0x7f42bc1018b0 2024-04-06T14:00:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69515]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69515]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69515]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:55 smithi090 sudo[69515]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58644]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58644]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58644]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:55 smithi134 sudo[58644]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 sudo[58647]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 sudo[58647]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 sudo[58647]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:56.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 sudo[58647]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:56.542 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57548]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57548]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57548]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57548]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57551]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57551]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57551]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-06T14:00:56.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 sudo[57551]: pam_unix(sudo:session): session closed for user root 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: mgrmap e16: x(active, since 111s), standbys: z, y 2024-04-06T14:00:56.828 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.829 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: mgrmap e16: x(active, since 111s), standbys: z, y 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: mgrmap e16: x(active, since 111s), standbys: z, y 2024-04-06T14:00:56.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:00:56.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:00:57.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:57 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:57.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:57 smithi090 ceph-mon[43420]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:57.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:57 smithi090 ceph-mon[43420]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:57.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:57 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:57.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:57 smithi134 ceph-mon[47549]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:57.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:57 smithi134 ceph-mon[47549]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:57.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:57 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-06T14:00:57.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:57 smithi105 ceph-mon[46969]: pgmap v68: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:57.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:57 smithi105 ceph-mon[46969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-06T14:00:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:58 smithi134 ceph-mon[47549]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-06T14:00:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:00:58 smithi134 ceph-mon[47549]: Deploying daemon osd.3 on smithi090 2024-04-06T14:00:58.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:58 smithi090 ceph-mon[43420]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:58.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-06T14:00:58.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:58.920 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:00:58 smithi090 ceph-mon[43420]: Deploying daemon osd.3 on smithi090 2024-04-06T14:00:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:58 smithi105 ceph-mon[46969]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:00:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-06T14:00:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:00:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:00:58 smithi105 ceph-mon[46969]: Deploying daemon osd.3 on smithi090 2024-04-06T14:01:01.354 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:01 smithi090 ceph-mon[43420]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:01 smithi105 ceph-mon[46969]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:01.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:01 smithi134 ceph-mon[47549]: pgmap v70: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:02 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:02 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:02 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:02 smithi090 ceph-mon[43420]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-06T14:01:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:02 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:02 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:02 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:02 smithi134 ceph-mon[47549]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-06T14:01:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:02 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:02 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:02 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:02 smithi105 ceph-mon[46969]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 44 KiB/s wr, 4 op/s 2024-04-06T14:01:04.119 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 3 on host 'smithi090' 2024-04-06T14:01:04.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.118+0000 7f42a3fff700 1 -- 172.21.15.90:0/3989048616 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f42bc068e70 con 0x7f42a40a1110 2024-04-06T14:01:04.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 msgr2=0x7f42a40a35d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 0x7f42a40a35d0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f42ac003820 tx=0x7f42ac005ec0 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 msgr2=0x7f42bc102830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f42b400b970 tx=0x7f42b400bc80 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 shutdown_connections 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f42a40a1110 0x7f42a40a35d0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f42bc1018b0 0x7f42bc102830 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f42bc0ffd10 0x7f42bc103c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 --2- 172.21.15.90:0/3989048616 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42bc100f10 0x7f42bc1022f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 >> 172.21.15.90:0/3989048616 conn(0x7f42bc0fb2b0 msgr2=0x7f42bc068760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 shutdown_connections 2024-04-06T14:01:04.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:04.121+0000 7f42c1027700 1 -- 172.21.15.90:0/3989048616 wait complete. 2024-04-06T14:01:04.287 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.287 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.287 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:04.287 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:04.287 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.496 DEBUG:teuthology.orchestra.run.smithi090:osd.3> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.3.service 2024-04-06T14:01:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:04.499 INFO:tasks.cephadm:Deploying osd.4 on smithi105 with /dev/vg_nvme/lv_3... 2024-04-06T14:01:04.500 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-06T14:01:04.532 INFO:journalctl@ceph.osd.3.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:01:04.732 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:01:05.137 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:05.138 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:05 smithi105 ceph-mon[46969]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-06T14:01:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:05 smithi090 ceph-mon[43420]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-06T14:01:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:05 smithi134 ceph-mon[47549]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 33 KiB/s wr, 3 op/s 2024-04-06T14:01:05.763 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:01:05 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:01:05.761+0000 7f5665461640 -1 osd.3 0 log_to_monitors true 2024-04-06T14:01:06.010 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-06T14:01:06.011 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-06T14:01:06.011 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-04-06T14:01:06.011 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-04-06T14:01:06.011 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112968 s, 928 MB/s 2024-04-06T14:01:06.011 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-04-06T14:01:06.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:06 smithi105 ceph-mon[46969]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.307 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:06 smithi105 ceph-mon[46969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:06 smithi134 ceph-mon[47549]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:06 smithi134 ceph-mon[47549]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.399 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi105:vg_nvme/lv_3 2024-04-06T14:01:06.411 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:06 smithi090 ceph-mon[43420]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.411 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:06 smithi090 ceph-mon[43420]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-06T14:01:06.618 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:01:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-06T14:01:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: osdmap e23: 4 total, 3 up, 4 in 2024-04-06T14:01:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-06T14:01:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:07.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:07.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.397 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-06T14:01:07.397 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: osdmap e23: 4 total, 3 up, 4 in 2024-04-06T14:01:07.397 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:07.397 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-06T14:01:07.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.398 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:07.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:07.399 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-06T14:01:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: osdmap e23: 4 total, 3 up, 4 in 2024-04-06T14:01:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail; 1023 B/s rd, 26 KiB/s wr, 2 op/s 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:07.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 -- 172.21.15.105:0/2986293829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c0ffd10 msgr2=0x7f7d6c100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:07.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 --2- 172.21.15.105:0/2986293829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c100130 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f7d600099c0 tx=0x7f7d60009cd0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 -- 172.21.15.105:0/2986293829 shutdown_connections 2024-04-06T14:01:07.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 --2- 172.21.15.105:0/2986293829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c1018b0 0x7f7d6c108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.593 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 --2- 172.21.15.105:0/2986293829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c100130 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.593 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 --2- 172.21.15.105:0/2986293829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c100f10 0x7f7d6c101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.593 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 -- 172.21.15.105:0/2986293829 >> 172.21.15.105:0/2986293829 conn(0x7f7d6c0fb2b0 msgr2=0x7f7d6c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:07.593 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 -- 172.21.15.105:0/2986293829 shutdown_connections 2024-04-06T14:01:07.593 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.592+0000 7f7d74301700 1 -- 172.21.15.105:0/2986293829 wait complete. 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 Processor -- start 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 -- start start 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c195cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c1018b0 0x7f7d6c19d1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c19d7c0 con 0x7f7d6c0ffd10 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c19d900 con 0x7f7d6c100f10 2024-04-06T14:01:07.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d74301700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7d6c19dac0 con 0x7f7d6c1018b0 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7209d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c195cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:49004/0 (socket says 172.21.15.105:49004) 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 -- 172.21.15.105:0/3738552802 learned_addr learned my addr 172.21.15.105:0/3738552802 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7289e700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c1018b0 0x7f7d6c19d1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 -- 172.21.15.105:0/3738552802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c1018b0 msgr2=0x7f7d6c19d1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c1018b0 0x7f7d6c19d1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 -- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 msgr2=0x7f7d6c195cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.593+0000 7f7d7189c700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c195cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d7189c700 1 -- 172.21.15.105:0/3738552802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d600095f0 con 0x7f7d6c100f10 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d7209d700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c195cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d7189c700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f7d6800d980 tx=0x7f7d6800dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d68004220 con 0x7f7d6c100f10 2024-04-06T14:01:07.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7d6c19dda0 con 0x7f7d6c100f10 2024-04-06T14:01:07.597 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7d68007640 con 0x7f7d6c100f10 2024-04-06T14:01:07.597 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7d68005690 con 0x7f7d6c100f10 2024-04-06T14:01:07.597 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.594+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7d6c19e1c0 con 0x7f7d6c100f10 2024-04-06T14:01:07.597 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.595+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7d6c1071e0 con 0x7f7d6c100f10 2024-04-06T14:01:07.598 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.596+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f7d68004750 con 0x7f7d6c100f10 2024-04-06T14:01:07.598 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.596+0000 7f7d5f7fe700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 0x7f7d580a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:07.598 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.596+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(25..25 src has 1..25) v4 ==== 3422+0+0 (secure 0 0 0) 0x7f7d680cb3d0 con 0x7f7d6c100f10 2024-04-06T14:01:07.598 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.596+0000 7f7d7209d700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 0x7f7d580a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:07.598 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.597+0000 7f7d7209d700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 0x7f7d580a35e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7d60009990 tx=0x7f7d6001a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:07.601 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.601+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7d68095e80 con 0x7f7d6c100f10 2024-04-06T14:01:07.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:07.799+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f7d6c02cdc0 con 0x7f7d580a1120 2024-04-06T14:01:08.038 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:01:07.584+0000 7f565aac5700 -1 osd.3 0 waiting for initial osdmap 2024-04-06T14:01:08.038 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:01:07 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:01:07.590+0000 7f56528b2700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi090 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 8556M 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: osdmap e24: 4 total, 3 up, 4 in 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: osdmap e25: 4 total, 3 up, 4 in 2024-04-06T14:01:08.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='osd.3 ' entity='osd.3' 2024-04-06T14:01:08.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:08.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:08.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi090 2024-04-06T14:01:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 8556M 2024-04-06T14:01:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: osdmap e24: 4 total, 3 up, 4 in 2024-04-06T14:01:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: osdmap e25: 4 total, 3 up, 4 in 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='osd.3 ' entity='osd.3' 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:08.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi090 2024-04-06T14:01:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 8556M 2024-04-06T14:01:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: osdmap e24: 4 total, 3 up, 4 in 2024-04-06T14:01:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: osdmap e25: 4 total, 3 up, 4 in 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='osd.3 ' entity='osd.3' 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:01:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:01:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: from='client.24221 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:09.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:09.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] boot 2024-04-06T14:01:09.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: osdmap e26: 4 total, 4 up, 4 in 2024-04-06T14:01:09.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:09 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: from='client.24221 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] boot 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: osdmap e26: 4 total, 4 up, 4 in 2024-04-06T14:01:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:09 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:09.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:01:09.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:01:09.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: from='client.24221 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:09.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:09.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-06T14:01:09.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: osd.3 [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] boot 2024-04-06T14:01:09.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: osdmap e26: 4 total, 4 up, 4 in 2024-04-06T14:01:09.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:09 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:01:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: osdmap e27: 4 total, 4 up, 4 in 2024-04-06T14:01:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/3619897975' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]': finished 2024-04-06T14:01:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: osdmap e28: 5 total, 4 up, 5 in 2024-04-06T14:01:10.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:10.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:10.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:10 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/2880317526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: osdmap e27: 4 total, 4 up, 4 in 2024-04-06T14:01:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/3619897975' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]': finished 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: osdmap e28: 5 total, 4 up, 5 in 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:10 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/2880317526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: osdmap e27: 4 total, 4 up, 4 in 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/3619897975' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]: dispatch 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb3d511-2b64-45cb-845e-57589ef3fb59"}]': finished 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: osdmap e28: 5 total, 4 up, 5 in 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:10 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/2880317526' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:11 smithi134 ceph-mon[47549]: osdmap e29: 5 total, 4 up, 5 in 2024-04-06T14:01:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:11 smithi105 ceph-mon[46969]: osdmap e29: 5 total, 4 up, 5 in 2024-04-06T14:01:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:11 smithi090 ceph-mon[43420]: osdmap e29: 5 total, 4 up, 5 in 2024-04-06T14:01:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:12 smithi134 ceph-mon[47549]: osdmap e30: 5 total, 4 up, 5 in 2024-04-06T14:01:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:12 smithi134 ceph-mon[47549]: pgmap v84: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 157 KiB/s, 0 objects/s recovering 2024-04-06T14:01:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:12 smithi105 ceph-mon[46969]: osdmap e30: 5 total, 4 up, 5 in 2024-04-06T14:01:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:12 smithi105 ceph-mon[46969]: pgmap v84: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 157 KiB/s, 0 objects/s recovering 2024-04-06T14:01:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:12 smithi090 ceph-mon[43420]: osdmap e30: 5 total, 4 up, 5 in 2024-04-06T14:01:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:12 smithi090 ceph-mon[43420]: pgmap v84: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 157 KiB/s, 0 objects/s recovering 2024-04-06T14:01:15.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:15 smithi105 ceph-mon[46969]: pgmap v85: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:01:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:15 smithi090 ceph-mon[43420]: pgmap v85: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:01:15.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:15 smithi134 ceph-mon[47549]: pgmap v85: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:01:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-06T14:01:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-06T14:01:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:16.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-06T14:01:16.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:17.376 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:17 smithi105 ceph-mon[46969]: Deploying daemon osd.4 on smithi105 2024-04-06T14:01:17.376 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:17 smithi105 ceph-mon[46969]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-06T14:01:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:17 smithi090 ceph-mon[43420]: Deploying daemon osd.4 on smithi105 2024-04-06T14:01:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:17 smithi090 ceph-mon[43420]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-06T14:01:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:17 smithi134 ceph-mon[47549]: Deploying daemon osd.4 on smithi105 2024-04-06T14:01:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:17 smithi134 ceph-mon[47549]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-06T14:01:18.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:18 smithi105 ceph-mon[46969]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:01:18.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:18 smithi090 ceph-mon[43420]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:01:18.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:18 smithi134 ceph-mon[47549]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:01:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:20.488 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.488 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.488 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:21.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:21.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:21.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:21.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.399 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 4 on host 'smithi105' 2024-04-06T14:01:21.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.399+0000 7f7d5f7fe700 1 -- 172.21.15.105:0/3738552802 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f7d6c02cdc0 con 0x7f7d580a1120 2024-04-06T14:01:21.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 msgr2=0x7f7d580a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:21.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 0x7f7d580a35e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7d60009990 tx=0x7f7d6001a040 comp rx=0 tx=0).stop 2024-04-06T14:01:21.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 msgr2=0x7f7d6c196200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f7d6800d980 tx=0x7f7d6800dc90 comp rx=0 tx=0).stop 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 shutdown_connections 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7d580a1120 0x7f7d580a35e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7d6c1018b0 0x7f7d6c19d1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7d6c0ffd10 0x7f7d6c195cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 --2- 172.21.15.105:0/3738552802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7d6c100f10 0x7f7d6c196200 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 >> 172.21.15.105:0/3738552802 conn(0x7f7d6c0fb2b0 msgr2=0x7f7d6c104320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 shutdown_connections 2024-04-06T14:01:21.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:21.401+0000 7f7d74301700 1 -- 172.21.15.105:0/3738552802 wait complete. 2024-04-06T14:01:21.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:21.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.422 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:21.423 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:21.423 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:21.779 DEBUG:teuthology.orchestra.run.smithi105:osd.4> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.4.service 2024-04-06T14:01:21.781 INFO:tasks.cephadm:Deploying osd.5 on smithi134 with /dev/vg_nvme/lv_3... 2024-04-06T14:01:21.781 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-06T14:01:21.813 INFO:journalctl@ceph.osd.4.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:01:22.016 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:01:22.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:22 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:22 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:22 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:22.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:22 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:22 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:22 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:22.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:22 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:22 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:22.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:22 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:23.267 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-06T14:01:23.267 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-06T14:01:23.268 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-04-06T14:01:23.268 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-04-06T14:01:23.268 INFO:teuthology.orchestra.run.smithi134.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0154384 s, 679 MB/s 2024-04-06T14:01:23.268 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-04-06T14:01:23.485 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:23 smithi105 ceph-mon[46969]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:23.485 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:23 smithi105 ceph-mon[46969]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.485 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:23 smithi105 ceph-mon[46969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.485 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:01:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:01:23.132+0000 7f3f86974640 -1 osd.4 0 log_to_monitors true 2024-04-06T14:01:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:23 smithi134 ceph-mon[47549]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:23 smithi134 ceph-mon[47549]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:23 smithi134 ceph-mon[47549]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.673 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi134:vg_nvme/lv_3 2024-04-06T14:01:23.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:23 smithi090 ceph-mon[43420]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:23.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:23 smithi090 ceph-mon[43420]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:23 smithi090 ceph-mon[43420]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-06T14:01:23.894 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:01:24.578 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-06T14:01:24.578 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: osdmap e31: 5 total, 4 up, 5 in 2024-04-06T14:01:24.578 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:24.578 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.578 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi105 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 8557M 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.579 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:24 smithi134 ceph-mon[47549]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:24.748 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 -- 172.21.15.134:0/965115706 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92000ffd10 msgr2=0x7f9200100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:24.748 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 --2- 172.21.15.134:0/965115706 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92000ffd10 0x7f9200100130 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f91f0009990 tx=0x7f91f0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.748 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 -- 172.21.15.134:0/965115706 shutdown_connections 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 --2- 172.21.15.134:0/965115706 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92001018b0 0x7f9200108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 --2- 172.21.15.134:0/965115706 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92000ffd10 0x7f9200100130 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.746+0000 7f9205f65700 1 --2- 172.21.15.134:0/965115706 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 0x7f9200101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.747+0000 7f9205f65700 1 -- 172.21.15.134:0/965115706 >> 172.21.15.134:0/965115706 conn(0x7f92000fb2b0 msgr2=0x7f92000fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.747+0000 7f9205f65700 1 -- 172.21.15.134:0/965115706 shutdown_connections 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.747+0000 7f9205f65700 1 -- 172.21.15.134:0/965115706 wait complete. 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.747+0000 7f9205f65700 1 Processor -- start 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.747+0000 7f9205f65700 1 -- start start 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92000ffd10 0x7f92000691d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 0x7f920019bbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019e890 con 0x7f9200100f10 2024-04-06T14:01:24.749 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019ea00 con 0x7f92000ffd10 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019ebc0 con 0x7f92001018b0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91ff7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92000ffd10 0x7f92000691d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:37538/0 (socket says 172.21.15.134:37538) 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 -- 172.21.15.134:0/1023074792 learned_addr learned my addr 172.21.15.134:0/1023074792 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91feffd700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 0x7f920019bbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 -- 172.21.15.134:0/1023074792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92000ffd10 msgr2=0x7f92000691d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92000ffd10 0x7f92000691d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 -- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 msgr2=0x7f920019bbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 0x7f920019bbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 -- 172.21.15.134:0/1023074792 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91f00095f0 con 0x7f92001018b0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fffff700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f91f800b970 tx=0x7f91f800bc80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:24.750 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91f800d860 con 0x7f92001018b0 2024-04-06T14:01:24.751 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91f800de80 con 0x7f92001018b0 2024-04-06T14:01:24.751 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f920019eea0 con 0x7f92001018b0 2024-04-06T14:01:24.751 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91f8017400 con 0x7f92001018b0 2024-04-06T14:01:24.751 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.748+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f920019f310 con 0x7f92001018b0 2024-04-06T14:01:24.751 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:01:24.460+0000 7f3f7bfd8700 -1 osd.4 0 waiting for initial osdmap 2024-04-06T14:01:24.752 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:01:24.467+0000 7f3f77dcd700 -1 osd.4 32 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: osdmap e31: 5 total, 4 up, 5 in 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi105 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.752 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 8557M 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.753 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:24 smithi105 ceph-mon[46969]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:24.753 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.750+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9200107240 con 0x7f92001018b0 2024-04-06T14:01:24.754 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.750+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f91f800e630 con 0x7f92001018b0 2024-04-06T14:01:24.754 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.751+0000 7f91fcff9700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 0x7f91e80a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:24.754 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.751+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(32..32 src has 1..32) v4 ==== 3821+0+0 (secure 0 0 0) 0x7f91f8006e40 con 0x7f92001018b0 2024-04-06T14:01:24.754 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.751+0000 7f91ff7fe700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 0x7f91e80a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:24.754 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.752+0000 7f91ff7fe700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 0x7f91e80a35e0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f91f0003790 tx=0x7f91f0005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:24.755 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.755+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f91f8095c10 con 0x7f92001018b0 2024-04-06T14:01:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-06T14:01:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: osdmap e31: 5 total, 4 up, 5 in 2024-04-06T14:01:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi105 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 8557M 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:24 smithi090 ceph-mon[43420]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:24.954 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:24.954+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f920002cd20 con 0x7f91e80a1120 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: osdmap e32: 5 total, 4 up, 5 in 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:25.610 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:25.611 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: osdmap e32: 5 total, 4 up, 5 in 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:25.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:25.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:01:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: osdmap e32: 5 total, 4 up, 5 in 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='client.24238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] boot 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: osdmap e33: 5 total, 5 up, 5 in 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:26 smithi105 ceph-mon[46969]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:01:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:01:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] boot 2024-04-06T14:01:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: osdmap e33: 5 total, 5 up, 5 in 2024-04-06T14:01:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:26 smithi090 ceph-mon[43420]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:01:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:01:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: osd.4 [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] boot 2024-04-06T14:01:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: osdmap e33: 5 total, 5 up, 5 in 2024-04-06T14:01:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:01:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:26 smithi134 ceph-mon[47549]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 111 MiB used, 358 GiB / 358 GiB avail 2024-04-06T14:01:27.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: osdmap e34: 5 total, 5 up, 5 in 2024-04-06T14:01:27.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/592530949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]': finished 2024-04-06T14:01:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: osdmap e35: 6 total, 5 up, 6 in 2024-04-06T14:01:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:27 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/2435490824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: osdmap e34: 5 total, 5 up, 5 in 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/592530949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]': finished 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: osdmap e35: 6 total, 5 up, 6 in 2024-04-06T14:01:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:27.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:27 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/2435490824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: osdmap e34: 5 total, 5 up, 5 in 2024-04-06T14:01:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/592530949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]: dispatch 2024-04-06T14:01:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "989a2709-b8f9-47e7-9385-54bc5d4287ac"}]': finished 2024-04-06T14:01:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: osdmap e35: 6 total, 5 up, 6 in 2024-04-06T14:01:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:27 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/2435490824' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:28.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:28 smithi105 ceph-mon[46969]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:28.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:28 smithi090 ceph-mon[43420]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:28 smithi134 ceph-mon[47549]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:31.367 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:31 smithi134 ceph-mon[47549]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:31 smithi105 ceph-mon[46969]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:31 smithi090 ceph-mon[43420]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 137 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:33 smithi134 ceph-mon[47549]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-06T14:01:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:33 smithi105 ceph-mon[46969]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-06T14:01:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:33 smithi090 ceph-mon[43420]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-06T14:01:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:34.465 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:34 smithi134 ceph-mon[47549]: Deploying daemon osd.5 on smithi134 2024-04-06T14:01:34.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:34 smithi105 ceph-mon[46969]: Deploying daemon osd.5 on smithi134 2024-04-06T14:01:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:34 smithi090 ceph-mon[43420]: Deploying daemon osd.5 on smithi134 2024-04-06T14:01:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:35 smithi105 ceph-mon[46969]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:35 smithi090 ceph-mon[43420]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:35 smithi134 ceph-mon[47549]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:36.488 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:36.488 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:36.488 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:37 smithi134 ceph-mon[47549]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:37 smithi105 ceph-mon[46969]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:37 smithi090 ceph-mon[43420]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:38.013 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.012+0000 7f91fcff9700 1 -- 172.21.15.134:0/1023074792 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f920002cd20 con 0x7f91e80a1120 2024-04-06T14:01:38.014 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 5 on host 'smithi134' 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 msgr2=0x7f91e80a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 0x7f91e80a35e0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f91f0003790 tx=0x7f91f0005fd0 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 msgr2=0x7f920019e110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f91f800b970 tx=0x7f91f800bc80 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 shutdown_connections 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91e80a1120 0x7f91e80a35e0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f92001018b0 0x7f920019e110 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f92000ffd10 0x7f92000691d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 --2- 172.21.15.134:0/1023074792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9200100f10 0x7f920019bbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 >> 172.21.15.134:0/1023074792 conn(0x7f92000fb2b0 msgr2=0x7f9200104380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:38.016 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 shutdown_connections 2024-04-06T14:01:38.017 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:01:38.015+0000 7f9205f65700 1 -- 172.21.15.134:0/1023074792 wait complete. 2024-04-06T14:01:38.388 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.388 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.388 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:38.388 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:38.388 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.389 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.389 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.389 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:38.390 DEBUG:teuthology.orchestra.run.smithi134:osd.5> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.5.service 2024-04-06T14:01:38.392 INFO:tasks.cephadm:Deploying osd.6 on smithi090 with /dev/vg_nvme/lv_2... 2024-04-06T14:01:38.392 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-06T14:01:38.421 INFO:journalctl@ceph.osd.5.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:01:38.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:38.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:38.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:38.624 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:39.387 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:39 smithi090 ceph-mon[43420]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:39.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:39 smithi134 ceph-mon[47549]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:39.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:39 smithi105 ceph-mon[46969]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0119747 s, 876 MB/s 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 2024-04-06T14:01:39.895 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-06T14:01:40.202 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:01:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:01:39.929+0000 7f1f847e8640 -1 osd.5 0 log_to_monitors true 2024-04-06T14:01:40.292 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi090:vg_nvme/lv_2 2024-04-06T14:01:40.520 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:01:40.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:40 smithi090 ceph-mon[43420]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-06T14:01:40.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:40 smithi134 ceph-mon[47549]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-06T14:01:40.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:40 smithi105 ceph-mon[46969]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-06T14:01:41.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 -- 172.21.15.90:0/2106418848 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c104270 msgr2=0x7f288c106660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:41.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 --2- 172.21.15.90:0/2106418848 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c104270 0x7f288c106660 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f28880098f0 tx=0x7f2888009c00 comp rx=0 tx=0).stop 2024-04-06T14:01:41.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 -- 172.21.15.90:0/2106418848 shutdown_connections 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 --2- 172.21.15.90:0/2106418848 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c106ba0 0x7f288c108f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 --2- 172.21.15.90:0/2106418848 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c101940 0x7f288c103d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.479+0000 7f2893518700 1 --2- 172.21.15.90:0/2106418848 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c104270 0x7f288c106660 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- 172.21.15.90:0/2106418848 >> 172.21.15.90:0/2106418848 conn(0x7f288c0fb2d0 msgr2=0x7f288c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- 172.21.15.90:0/2106418848 shutdown_connections 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- 172.21.15.90:0/2106418848 wait complete. 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 Processor -- start 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- start start 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c104270 0x7f288c1994f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c106ba0 0x7f288c1a0520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f288c1a0af0 con 0x7f288c101940 2024-04-06T14:01:41.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f288c1a0c30 con 0x7f288c104270 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f2893518700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f288c1a0dc0 con 0x7f288c106ba0 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f28912b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f28912b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54826/0 (socket says 172.21.15.90:54826) 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.480+0000 7f28912b4700 1 -- 172.21.15.90:0/1593983090 learned_addr learned my addr 172.21.15.90:0/1593983090 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f2891ab5700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c106ba0 0x7f288c1a0520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f2890ab3700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c104270 0x7f288c1994f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 -- 172.21.15.90:0/1593983090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c106ba0 msgr2=0x7f288c1a0520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c106ba0 0x7f288c1a0520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 -- 172.21.15.90:0/1593983090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c104270 msgr2=0x7f288c1994f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c104270 0x7f288c1994f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 -- 172.21.15.90:0/1593983090 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2888009520 con 0x7f288c101940 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28912b4700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f287c00e9f0 tx=0x7f287c00ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:41.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f287c004a30 con 0x7f288c101940 2024-04-06T14:01:41.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f288c1a10a0 con 0x7f288c101940 2024-04-06T14:01:41.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f287c0054e0 con 0x7f288c101940 2024-04-06T14:01:41.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f287c00cc80 con 0x7f288c101940 2024-04-06T14:01:41.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.481+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f288c1a15f0 con 0x7f288c101940 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.482+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f288c04ea80 con 0x7f288c101940 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.484+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f287c005000 con 0x7f288c101940 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.484+0000 7f28827fc700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 0x7f28780a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.484+0000 7f2890ab3700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 0x7f28780a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.484+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(37..37 src has 1..37) v4 ==== 4253+0+0 (secure 0 0 0) 0x7f287c014070 con 0x7f288c101940 2024-04-06T14:01:41.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.485+0000 7f2890ab3700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 0x7f28780a35e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2888005e90 tx=0x7f2888019040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:41.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.488+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f287c096630 con 0x7f288c101940 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: osdmap e36: 6 total, 5 up, 6 in 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi134 2024-04-06T14:01:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 8557M 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:41.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:41.687+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f288c100060 con 0x7f28780a1120 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: osdmap e36: 6 total, 5 up, 6 in 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi134 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 8557M 2024-04-06T14:01:41.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:41.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:41.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:41.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: osdmap e36: 6 total, 5 up, 6 in 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi134 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 8557M 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:41.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:42.143 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:01:41.739+0000 7f1f7a64d700 -1 osd.5 0 waiting for initial osdmap 2024-04-06T14:01:42.143 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:01:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:01:41.746+0000 7f1f71c39700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: osdmap e37: 6 total, 5 up, 6 in 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:42 smithi134 ceph-mon[47549]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' 2024-04-06T14:01:42.735 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: osdmap e37: 6 total, 5 up, 6 in 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:42.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:42 smithi090 ceph-mon[43420]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: osdmap e37: 6 total, 5 up, 6 in 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:42.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:42 smithi105 ceph-mon[46969]: from='osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087]' entity='osd.5' 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: from='client.14370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] boot 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: osdmap e38: 6 total, 6 up, 6 in 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:43 smithi134 ceph-mon[47549]: osdmap e39: 6 total, 6 up, 6 in 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: from='client.14370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] boot 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: osdmap e38: 6 total, 6 up, 6 in 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:43 smithi105 ceph-mon[46969]: osdmap e39: 6 total, 6 up, 6 in 2024-04-06T14:01:43.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:01:43.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: from='client.14370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 138 MiB used, 447 GiB / 447 GiB avail 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: osd.5 [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] boot 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: osdmap e38: 6 total, 6 up, 6 in 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:01:43.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:43 smithi090 ceph-mon[43420]: osdmap e39: 6 total, 6 up, 6 in 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: osdmap e40: 6 total, 6 up, 6 in 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]: dispatch 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]': finished 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: osdmap e41: 7 total, 6 up, 7 in 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 168 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:44 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2645216585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:44.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: osdmap e40: 6 total, 6 up, 6 in 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]: dispatch 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]': finished 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: osdmap e41: 7 total, 6 up, 7 in 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 168 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:44.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:44 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2645216585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:44.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: osdmap e40: 6 total, 6 up, 6 in 2024-04-06T14:01:44.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]: dispatch 2024-04-06T14:01:44.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3192046248' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fe9a9e4-959e-4fd9-b50d-701606e38f79"}]': finished 2024-04-06T14:01:44.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: osdmap e41: 7 total, 6 up, 7 in 2024-04-06T14:01:44.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:44.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 168 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:44.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:44 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2645216585' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:01:45.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:45 smithi090 ceph-mon[43420]: osdmap e42: 7 total, 6 up, 7 in 2024-04-06T14:01:45.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:45.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:45 smithi134 ceph-mon[47549]: osdmap e42: 7 total, 6 up, 7 in 2024-04-06T14:01:45.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:45.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:45 smithi105 ceph-mon[46969]: osdmap e42: 7 total, 6 up, 7 in 2024-04-06T14:01:45.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:46.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:46 smithi090 ceph-mon[43420]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 150 KiB/s, 0 objects/s recovering 2024-04-06T14:01:46.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:46 smithi134 ceph-mon[47549]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 150 KiB/s, 0 objects/s recovering 2024-04-06T14:01:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:46 smithi105 ceph-mon[46969]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 150 KiB/s, 0 objects/s recovering 2024-04-06T14:01:48.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:48 smithi090 ceph-mon[43420]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-06T14:01:48.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:48 smithi105 ceph-mon[46969]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-06T14:01:48.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:48 smithi134 ceph-mon[47549]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-06T14:01:49.733 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-06T14:01:49.733 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:49.733 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:49 smithi090 ceph-mon[43420]: Deploying daemon osd.6 on smithi090 2024-04-06T14:01:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-06T14:01:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:49 smithi105 ceph-mon[46969]: Deploying daemon osd.6 on smithi090 2024-04-06T14:01:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-06T14:01:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:49 smithi134 ceph-mon[47549]: Deploying daemon osd.6 on smithi090 2024-04-06T14:01:50.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:50 smithi105 ceph-mon[46969]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-06T14:01:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:50 smithi090 ceph-mon[43420]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-06T14:01:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:50 smithi134 ceph-mon[47549]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-06T14:01:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:53 smithi090 ceph-mon[43420]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-04-06T14:01:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:53 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:53 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:53 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:53 smithi105 ceph-mon[46969]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-04-06T14:01:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:53 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:53 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:53 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:53 smithi134 ceph-mon[47549]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-04-06T14:01:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:53 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:53 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:53 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:55.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.083+0000 7f28827fc700 1 -- 172.21.15.90:0/1593983090 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f288c100060 con 0x7f28780a1120 2024-04-06T14:01:55.083 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 6 on host 'smithi090' 2024-04-06T14:01:55.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 msgr2=0x7f28780a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:55.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 0x7f28780a35e0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2888005e90 tx=0x7f2888019040 comp rx=0 tx=0).stop 2024-04-06T14:01:55.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 msgr2=0x7f288c198fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f287c00e9f0 tx=0x7f287c00ed00 comp rx=0 tx=0).stop 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 shutdown_connections 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28780a1120 0x7f28780a35e0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f288c106ba0 0x7f288c1a0520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f288c101940 0x7f288c198fb0 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 --2- 172.21.15.90:0/1593983090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f288c104270 0x7f288c1994f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.085+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 >> 172.21.15.90:0/1593983090 conn(0x7f288c0fb2d0 msgr2=0x7f288c0ff950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.086+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 shutdown_connections 2024-04-06T14:01:55.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:01:55.086+0000 7f2893518700 1 -- 172.21.15.90:0/1593983090 wait complete. 2024-04-06T14:01:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-06T14:01:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:55.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:55.529 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-06T14:01:55.529 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.530 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:55.533 DEBUG:teuthology.orchestra.run.smithi090:osd.6> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.6.service 2024-04-06T14:01:55.535 INFO:tasks.cephadm:Deploying osd.7 on smithi105 with /dev/vg_nvme/lv_2... 2024-04-06T14:01:55.535 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-06T14:01:55.564 INFO:journalctl@ceph.osd.6.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:55.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:01:55.766 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112867 s, 929 MB/s 2024-04-06T14:01:57.093 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-04-06T14:01:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:57 smithi090 ceph-mon[43420]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-06T14:01:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:57 smithi090 ceph-mon[43420]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-06T14:01:57.289 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:01:57 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:01:57.023+0000 7fb459d5e640 -1 osd.6 0 log_to_monitors true 2024-04-06T14:01:57.391 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:57 smithi105 ceph-mon[46969]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-06T14:01:57.392 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:57 smithi105 ceph-mon[46969]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-06T14:01:57.485 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi105:vg_nvme/lv_2 2024-04-06T14:01:57.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:57 smithi134 ceph-mon[47549]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-06T14:01:57.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:57 smithi134 ceph-mon[47549]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-06T14:01:57.703 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:01:58.414 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: osdmap e43: 7 total, 6 up, 7 in 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:58.415 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-06T14:01:58.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: osdmap e43: 7 total, 6 up, 7 in 2024-04-06T14:01:58.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:58.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.625 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/122992189 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 msgr2=0x7f0d64100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/122992189 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d64100130 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f0d600099c0 tx=0x7f0d60009cd0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/122992189 shutdown_connections 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/122992189 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d64108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/122992189 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d64100130 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/122992189 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 0x7f0d64101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/122992189 >> 172.21.15.105:0/122992189 conn(0x7f0d640fb2d0 msgr2=0x7f0d640fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/122992189 shutdown_connections 2024-04-06T14:01:58.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.624+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/122992189 wait complete. 2024-04-06T14:01:58.628 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 Processor -- start 2024-04-06T14:01:58.628 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 -- start start 2024-04-06T14:01:58.628 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 0x7f0d6410c220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d6410cf00 con 0x7f0d640ffd10 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d6410d070 con 0x7f0d641018b0 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6c4b8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d641a0de0 con 0x7f0d64100f10 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6a254700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6a254700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.105:41990/0 (socket says 172.21.15.105:41990) 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:43486/0 (socket says 172.21.15.105:43486) 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6a254700 1 -- 172.21.15.105:0/921556130 learned_addr learned my addr 172.21.15.105:0/921556130 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d69a53700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 0x7f0d6410c220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 -- 172.21.15.105:0/921556130 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 msgr2=0x7f0d6410c220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:58.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 0x7f0d6410c220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 -- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 msgr2=0x7f0d6410db80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6aa55700 1 -- 172.21.15.105:0/921556130 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d600095f0 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.625+0000 7f0d6a254700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d6aa55700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0d5c00efc0 tx=0x7f0d5c00c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d5c018070 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d5c00f040 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d5c00cdf0 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d641a1010 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.626+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d641a14c0 con 0x7f0d641018b0 2024-04-06T14:01:58.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.627+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d6404ea80 con 0x7f0d641018b0 2024-04-06T14:01:58.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.628+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f0d5c003a00 con 0x7f0d641018b0 2024-04-06T14:01:58.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.628+0000 7f0d5b7fe700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 0x7f0d500a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:01:58.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.628+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(44..44 src has 1..44) v4 ==== 4685+0+0 (secure 0 0 0) 0x7f0d5c0cbce0 con 0x7f0d641018b0 2024-04-06T14:01:58.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.628+0000 7f0d6a254700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 0x7f0d500a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:01:58.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.629+0000 7f0d6a254700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 0x7f0d500a35e0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f0d60004800 tx=0x7f0d6001a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:01:58.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.632+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0d5c096330 con 0x7f0d641018b0 2024-04-06T14:01:58.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-06T14:01:58.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: osdmap e43: 7 total, 6 up, 7 in 2024-04-06T14:01:58.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:58.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:01:58.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:01:58.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:01:58.833 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:01:58.831+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f0d640fc0d0 con 0x7f0d500a1120 2024-04-06T14:01:59.038 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:01:58.628+0000 7fb44f3c2700 -1 osd.6 0 waiting for initial osdmap 2024-04-06T14:01:59.038 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:01:58 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:01:58.642+0000 7fb4471af700 -1 osd.6 44 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi090 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 5704M 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: osdmap e44: 7 total, 6 up, 7 in 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' 2024-04-06T14:01:59.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:59.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:59.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:01:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:59.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi090 2024-04-06T14:01:59.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 5704M 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: osdmap e44: 7 total, 6 up, 7 in 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:01:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi090 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 5704M 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: osdmap e44: 7 total, 6 up, 7 in 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:01:59.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412]' entity='osd.6' 2024-04-06T14:01:59.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:01:59.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:01:59.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:01:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] boot 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: osdmap e45: 7 total, 7 up, 7 in 2024-04-06T14:02:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:00.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:02:00.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:02:00.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:00.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:00.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] boot 2024-04-06T14:02:00.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: osdmap e45: 7 total, 7 up, 7 in 2024-04-06T14:02:00.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: osd.6 [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] boot 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: osdmap e45: 7 total, 7 up, 7 in 2024-04-06T14:02:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:02:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:02:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/2028914501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]': finished 2024-04-06T14:02:01.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: osdmap e46: 8 total, 7 up, 8 in 2024-04-06T14:02:01.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:01.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:02:01.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/2028914501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]': finished 2024-04-06T14:02:01.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: osdmap e46: 8 total, 7 up, 8 in 2024-04-06T14:02:01.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:01.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 169 MiB used, 536 GiB / 536 GiB avail 2024-04-06T14:02:01.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/2028914501' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]: dispatch 2024-04-06T14:02:01.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eef257f2-5d81-4b97-9186-03b54813ccea"}]': finished 2024-04-06T14:02:01.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: osdmap e46: 8 total, 7 up, 8 in 2024-04-06T14:02:01.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/3303115789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:02.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/3303115789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:02.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/3303115789' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:03 smithi105 ceph-mon[46969]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:03 smithi090 ceph-mon[43420]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:03.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:03 smithi134 ceph-mon[47549]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:05.379 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:05 smithi105 ceph-mon[46969]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:05.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:05 smithi090 ceph-mon[43420]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:05 smithi134 ceph-mon[47549]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:07 smithi105 ceph-mon[46969]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-06T14:02:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:07 smithi090 ceph-mon[43420]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-06T14:02:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:07 smithi134 ceph-mon[47549]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-06T14:02:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:08.299 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:08 smithi105 ceph-mon[46969]: Deploying daemon osd.7 on smithi105 2024-04-06T14:02:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:08 smithi090 ceph-mon[43420]: Deploying daemon osd.7 on smithi105 2024-04-06T14:02:08.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:08 smithi134 ceph-mon[47549]: Deploying daemon osd.7 on smithi105 2024-04-06T14:02:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:09 smithi105 ceph-mon[46969]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:09.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:09 smithi090 ceph-mon[43420]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:09.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:09 smithi134 ceph-mon[47549]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:11 smithi105 ceph-mon[46969]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:11 smithi090 ceph-mon[43420]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:11 smithi134 ceph-mon[47549]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:12.535 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 7 on host 'smithi105' 2024-04-06T14:02:12.535 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.534+0000 7f0d5b7fe700 1 -- 172.21.15.105:0/921556130 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f0d640fc0d0 con 0x7f0d500a1120 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 msgr2=0x7f0d500a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 0x7f0d500a35e0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f0d60004800 tx=0x7f0d6001a040 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 msgr2=0x7f0d6410c760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f0d5c00efc0 tx=0x7f0d5c00c3a0 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 shutdown_connections 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0d500a1120 0x7f0d500a35e0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d641018b0 0x7f0d6410c760 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d640ffd10 0x7f0d6410db80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 --2- 172.21.15.105:0/921556130 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d64100f10 0x7f0d6410c220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.536+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 >> 172.21.15.105:0/921556130 conn(0x7f0d640fb2d0 msgr2=0x7f0d640fb9c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.537+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 shutdown_connections 2024-04-06T14:02:12.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:12.537+0000 7f0d6c4b8700 1 -- 172.21.15.105:0/921556130 wait complete. 2024-04-06T14:02:12.908 DEBUG:teuthology.orchestra.run.smithi105:osd.7> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.7.service 2024-04-06T14:02:12.910 INFO:tasks.cephadm:Deploying osd.8 on smithi134 with /dev/vg_nvme/lv_2... 2024-04-06T14:02:12.911 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-06T14:02:12.943 INFO:journalctl@ceph.osd.7.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:12.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:12.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:13.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:13.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:13.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:13.146 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:02:14.398 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-06T14:02:14.399 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-06T14:02:14.399 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-04-06T14:02:14.399 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-04-06T14:02:14.399 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0150343 s, 697 MB/s 2024-04-06T14:02:14.399 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-04-06T14:02:14.729 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi134:vg_nvme/lv_2 2024-04-06T14:02:14.739 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:02:14 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:02:14.273+0000 7fbbed6da640 -1 osd.7 0 log_to_monitors true 2024-04-06T14:02:14.947 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:15.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:15.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:15 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.383 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.384 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.384 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.384 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:15.384 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:15.384 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:15 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:15.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:15.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:15 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:15.847 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 -- 172.21.15.134:0/602771313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb614100f10 msgr2=0x7fb614101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 --2- 172.21.15.134:0/602771313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb614100f10 0x7fb614101370 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fb604009990 tx=0x7fb604009ca0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 -- 172.21.15.134:0/602771313 shutdown_connections 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 --2- 172.21.15.134:0/602771313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb6141018b0 0x7fb614108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 --2- 172.21.15.134:0/602771313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6140ffd10 0x7fb614100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 --2- 172.21.15.134:0/602771313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb614100f10 0x7fb614101370 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 -- 172.21.15.134:0/602771313 >> 172.21.15.134:0/602771313 conn(0x7fb6140fb2b0 msgr2=0x7fb6140fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 -- 172.21.15.134:0/602771313 shutdown_connections 2024-04-06T14:02:15.848 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.847+0000 7fb61894f700 1 -- 172.21.15.134:0/602771313 wait complete. 2024-04-06T14:02:15.850 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 Processor -- start 2024-04-06T14:02:15.850 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 -- start start 2024-04-06T14:02:15.850 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 0x7fb61419ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 0x7fb61419e030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb61419e7d0 con 0x7fb6141018b0 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb61419e940 con 0x7fb614100f10 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61894f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb61419eb00 con 0x7fb6140ffd10 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.848+0000 7fb61259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:52802/0 (socket says 172.21.15.134:52802) 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 -- 172.21.15.134:0/1316912132 learned_addr learned my addr 172.21.15.134:0/1316912132 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb612d9d700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 0x7fb61419e030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb611d9b700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 0x7fb61419ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 -- 172.21.15.134:0/1316912132 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 msgr2=0x7fb61419ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 0x7fb61419ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.851 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 -- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 msgr2=0x7fb61419e030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 0x7fb61419e030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 -- 172.21.15.134:0/1316912132 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6040095f0 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb612d9d700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 0x7fb61419e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb611d9b700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 0x7fb61419ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61259c700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb60800d980 tx=0x7fb60800dd40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb60800de60 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb6080044e0 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb61419ede0 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb608003be0 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.849+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb61419f250 con 0x7fb6140ffd10 2024-04-06T14:02:15.852 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.850+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6141071f0 con 0x7fb6140ffd10 2024-04-06T14:02:15.853 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.851+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fb608009bb0 con 0x7fb6140ffd10 2024-04-06T14:02:15.853 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.852+0000 7fb6037fe700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 0x7fb5fc0a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:15.853 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.852+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 5101+0+0 (secure 0 0 0) 0x7fb6080cbcb0 con 0x7fb6140ffd10 2024-04-06T14:02:15.853 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.852+0000 7fb611d9b700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 0x7fb5fc0a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:15.853 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.853+0000 7fb611d9b700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 0x7fb5fc0a3660 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb60400b5c0 tx=0x7fb604005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:15.855 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:15.856+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb608096150 con 0x7fb6140ffd10 2024-04-06T14:02:16.054 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:16.053+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7fb61402cd90 con 0x7fb5fc0a11a0 2024-04-06T14:02:16.465 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi105 2024-04-06T14:02:16.465 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 5704M 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: osdmap e47: 8 total, 7 up, 8 in 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:16.466 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi105 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 5704M 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: osdmap e47: 8 total, 7 up, 8 in 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi105 2024-04-06T14:02:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 5704M 2024-04-06T14:02:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-06T14:02:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: osdmap e47: 8 total, 7 up, 8 in 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:16.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:16.997 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:02:16.685+0000 7fbbe2d3e700 -1 osd.7 0 waiting for initial osdmap 2024-04-06T14:02:16.997 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:02:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:02:16.696+0000 7fbbdab2b700 -1 osd.7 48 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: osdmap e48: 8 total, 7 up, 8 in 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='osd.7 ' entity='osd.7' 2024-04-06T14:02:17.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] boot 2024-04-06T14:02:17.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: osdmap e49: 8 total, 8 up, 8 in 2024-04-06T14:02:17.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:17 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: osdmap e48: 8 total, 7 up, 8 in 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='osd.7 ' entity='osd.7' 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] boot 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: osdmap e49: 8 total, 8 up, 8 in 2024-04-06T14:02:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:17 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='client.24313 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 200 MiB used, 626 GiB / 626 GiB avail 2024-04-06T14:02:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: osdmap e48: 8 total, 7 up, 8 in 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='osd.7 ' entity='osd.7' 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: osd.7 [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] boot 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: osdmap e49: 8 total, 8 up, 8 in 2024-04-06T14:02:17.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:17 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: osdmap e50: 8 total, 8 up, 8 in 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/2531057345' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]': finished 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: osdmap e51: 9 total, 8 up, 9 in 2024-04-06T14:02:18.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:18 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:18.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:02:18.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:02:18.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: osdmap e50: 8 total, 8 up, 8 in 2024-04-06T14:02:18.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/2531057345' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]': finished 2024-04-06T14:02:18.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: osdmap e51: 9 total, 8 up, 9 in 2024-04-06T14:02:18.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:18 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: osdmap e50: 8 total, 8 up, 8 in 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/2531057345' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]: dispatch 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "98c09ef0-8533-4ab9-b4ba-51aeb43ebc17"}]': finished 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: osdmap e51: 9 total, 8 up, 9 in 2024-04-06T14:02:18.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:18 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:19.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:19 smithi105 ceph-mon[46969]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:19.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:19 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/1843287305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:19.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:19 smithi090 ceph-mon[43420]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:19.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:19 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/1843287305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:19.647 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:19 smithi134 ceph-mon[47549]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:19.647 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:19 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/1843287305' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:20.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:20 smithi105 ceph-mon[46969]: osdmap e52: 9 total, 8 up, 9 in 2024-04-06T14:02:20.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:20 smithi090 ceph-mon[43420]: osdmap e52: 9 total, 8 up, 9 in 2024-04-06T14:02:20.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:20.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:20 smithi134 ceph-mon[47549]: osdmap e52: 9 total, 8 up, 9 in 2024-04-06T14:02:20.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:21.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:21 smithi105 ceph-mon[46969]: pgmap v140: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:21.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:21 smithi105 ceph-mon[46969]: osdmap e53: 9 total, 8 up, 9 in 2024-04-06T14:02:21.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:21 smithi090 ceph-mon[43420]: pgmap v140: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:21 smithi090 ceph-mon[43420]: osdmap e53: 9 total, 8 up, 9 in 2024-04-06T14:02:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:21.539 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:21 smithi134 ceph-mon[47549]: pgmap v140: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 232 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:21.539 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:21 smithi134 ceph-mon[47549]: osdmap e53: 9 total, 8 up, 9 in 2024-04-06T14:02:21.539 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:23.496 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:23 smithi134 ceph-mon[47549]: pgmap v142: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 233 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:23.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:23 smithi105 ceph-mon[46969]: pgmap v142: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 233 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:23.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:23 smithi090 ceph-mon[43420]: pgmap v142: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 233 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:24.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-06T14:02:24.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:24.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-06T14:02:24.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:24.626 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2024-04-06T14:02:24.627 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:25.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:25 smithi105 ceph-mon[46969]: Deploying daemon osd.8 on smithi134 2024-04-06T14:02:25.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:25 smithi105 ceph-mon[46969]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:25 smithi090 ceph-mon[43420]: Deploying daemon osd.8 on smithi134 2024-04-06T14:02:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:25 smithi090 ceph-mon[43420]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:25.625 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:25 smithi134 ceph-mon[47549]: Deploying daemon osd.8 on smithi134 2024-04-06T14:02:25.626 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:25 smithi134 ceph-mon[47549]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:27.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:27 smithi134 ceph-mon[47549]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:27.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:27 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:27 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:27 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:27 smithi090 ceph-mon[43420]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:27 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:27 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:27 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:27.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:27 smithi105 ceph-mon[46969]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:27 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:27 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:27.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:27 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:29.204 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.203+0000 7fb6037fe700 1 -- 172.21.15.134:0/1316912132 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fb61402cd90 con 0x7fb5fc0a11a0 2024-04-06T14:02:29.205 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 8 on host 'smithi134' 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 msgr2=0x7fb5fc0a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 0x7fb5fc0a3660 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb60400b5c0 tx=0x7fb604005fb0 comp rx=0 tx=0).stop 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 msgr2=0x7fb614069290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fb60800d980 tx=0x7fb60800dd40 comp rx=0 tx=0).stop 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 shutdown_connections 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb5fc0a11a0 0x7fb5fc0a3660 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6141018b0 0x7fb61419e030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:29.207 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6140ffd10 0x7fb614069290 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:29.208 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 --2- 172.21.15.134:0/1316912132 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb614100f10 0x7fb61419ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:29.208 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 >> 172.21.15.134:0/1316912132 conn(0x7fb6140fb2b0 msgr2=0x7fb614104330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:29.208 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 shutdown_connections 2024-04-06T14:02:29.208 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:02:29.206+0000 7fb61894f700 1 -- 172.21.15.134:0/1316912132 wait complete. 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.475 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.476 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:29 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:29 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:29 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:29.575 DEBUG:teuthology.orchestra.run.smithi134:osd.8> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.8.service 2024-04-06T14:02:29.577 INFO:tasks.cephadm:Deploying osd.9 on smithi090 with /dev/vg_nvme/lv_1... 2024-04-06T14:02:29.577 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-06T14:02:29.620 INFO:journalctl@ceph.osd.8.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:02:29.821 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:02:31.137 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-06T14:02:31.138 INFO:teuthology.orchestra.run.smithi090.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-06T14:02:31.138 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10+0 records in 2024-04-06T14:02:31.138 INFO:teuthology.orchestra.run.smithi090.stderr:10+0 records out 2024-04-06T14:02:31.138 INFO:teuthology.orchestra.run.smithi090.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112911 s, 929 MB/s 2024-04-06T14:02:31.138 INFO:teuthology.orchestra.run.smithi090.stderr:--> Zapping successful for: 2024-04-06T14:02:31.444 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:31 smithi090 ceph-mon[43420]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:31.445 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:31 smithi090 ceph-mon[43420]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.445 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:31 smithi090 ceph-mon[43420]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.458 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:02:31 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:02:31.151+0000 7f7c61a0a640 -1 osd.8 0 log_to_monitors true 2024-04-06T14:02:31.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:31 smithi134 ceph-mon[47549]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:31.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:31 smithi134 ceph-mon[47549]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:31 smithi134 ceph-mon[47549]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:31 smithi105 ceph-mon[46969]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:31 smithi105 ceph-mon[46969]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:31 smithi105 ceph-mon[46969]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2024-04-06T14:02:31.528 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi090:vg_nvme/lv_1 2024-04-06T14:02:31.756 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: osdmap e54: 9 total, 8 up, 9 in 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:32.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:32 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-06T14:02:32.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: osdmap e54: 9 total, 8 up, 9 in 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:32.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:32.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:32 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.643 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:02:32.234+0000 7f7c5706e700 -1 osd.8 0 waiting for initial osdmap 2024-04-06T14:02:32.643 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:02:32.248+0000 7f7c52e63700 -1 osd.8 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:02:32.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2024-04-06T14:02:32.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: osdmap e54: 9 total, 8 up, 9 in 2024-04-06T14:02:32.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:32.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:32.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:32.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:32 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:32.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 -- 172.21.15.90:0/778524079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda641018b0 msgr2=0x7fda64108160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:32.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 --2- 172.21.15.90:0/778524079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda641018b0 0x7fda64108160 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fda5c00b0d0 tx=0x7fda5c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 -- 172.21.15.90:0/778524079 shutdown_connections 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 --2- 172.21.15.90:0/778524079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda641018b0 0x7fda64108160 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 --2- 172.21.15.90:0/778524079 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 --2- 172.21.15.90:0/778524079 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda64100f10 0x7fda64101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 -- 172.21.15.90:0/778524079 >> 172.21.15.90:0/778524079 conn(0x7fda640fb2b0 msgr2=0x7fda640fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 -- 172.21.15.90:0/778524079 shutdown_connections 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.752+0000 7fda68d84700 1 -- 172.21.15.90:0/778524079 wait complete. 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 Processor -- start 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 -- start start 2024-04-06T14:02:32.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64195cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda641018b0 0x7fda6419d250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda6419d820 con 0x7fda64100f10 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda6419d960 con 0x7fda641018b0 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda68d84700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda6419db20 con 0x7fda640ffd10 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda61d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda61d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57400/0 (socket says 172.21.15.90:57400) 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.753+0000 7fda61d9b700 1 -- 172.21.15.90:0/1560184062 learned_addr learned my addr 172.21.15.90:0/1560184062 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda6259c700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64195cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda62d9d700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda641018b0 0x7fda6419d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 -- 172.21.15.90:0/1560184062 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 msgr2=0x7fda64195cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64195cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 -- 172.21.15.90:0/1560184062 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda641018b0 msgr2=0x7fda6419d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda641018b0 0x7fda6419d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 -- 172.21.15.90:0/1560184062 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda5c009d00 con 0x7fda64100f10 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda6259c700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64195cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda61d9b700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fda5400c920 tx=0x7fda5400cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda54007b00 con 0x7fda64100f10 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda54004cf0 con 0x7fda64100f10 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda54005610 con 0x7fda64100f10 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fda6419de00 con 0x7fda64100f10 2024-04-06T14:02:32.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.754+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda6419e220 con 0x7fda64100f10 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.755+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fda64107120 con 0x7fda64100f10 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.756+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fda54004750 con 0x7fda64100f10 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.757+0000 7fda537fe700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 0x7fda4c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.757+0000 7fda6259c700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 0x7fda4c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.757+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(56..56 src has 1..56) v4 ==== 5862+0+0 (secure 0 0 0) 0x7fda540cbec0 con 0x7fda64100f10 2024-04-06T14:02:32.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.757+0000 7fda6259c700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 0x7fda4c0a35e0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fda58005950 tx=0x7fda580058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:32.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.760+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fda54095fe0 con 0x7fda64100f10 2024-04-06T14:02:32.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:32.960+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7fda6402ce20 con 0x7fda4c0a1120 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi134 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 5704M 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: osdmap e55: 9 total, 8 up, 9 in 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] boot 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: osdmap e56: 9 total, 9 up, 9 in 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:33 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi134 2024-04-06T14:02:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 5704M 2024-04-06T14:02:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: osdmap e55: 9 total, 8 up, 9 in 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] boot 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: osdmap e56: 9 total, 9 up, 9 in 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:33 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi134 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 5704M 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: osdmap e55: 9 total, 8 up, 9 in 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: osd.8 [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] boot 2024-04-06T14:02:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: osdmap e56: 9 total, 9 up, 9 in 2024-04-06T14:02:33.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:02:33.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:33.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:33.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:33 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:34.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:34 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:02:34.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:34 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:02:34.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:34 smithi090 ceph-mon[43420]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:34.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:34 smithi090 ceph-mon[43420]: osdmap e57: 9 total, 9 up, 9 in 2024-04-06T14:02:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:34 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:02:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:34 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:02:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:34 smithi105 ceph-mon[46969]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:34 smithi105 ceph-mon[46969]: osdmap e57: 9 total, 9 up, 9 in 2024-04-06T14:02:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:34 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:02:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:34 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:02:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:34 smithi134 ceph-mon[47549]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi090:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:34 smithi134 ceph-mon[47549]: osdmap e57: 9 total, 9 up, 9 in 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: osdmap e58: 9 total, 9 up, 9 in 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]: dispatch 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]': finished 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: osdmap e59: 10 total, 9 up, 10 in 2024-04-06T14:02:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: osdmap e58: 9 total, 9 up, 9 in 2024-04-06T14:02:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]: dispatch 2024-04-06T14:02:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]': finished 2024-04-06T14:02:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: osdmap e59: 10 total, 9 up, 10 in 2024-04-06T14:02:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: osdmap e58: 9 total, 9 up, 9 in 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]: dispatch 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4156415791' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "599f84b9-90b8-4225-809c-96218058b7c4"}]': finished 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: osdmap e59: 10 total, 9 up, 10 in 2024-04-06T14:02:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1393136635' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:36 smithi105 ceph-mon[46969]: osdmap e60: 10 total, 9 up, 10 in 2024-04-06T14:02:36.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1393136635' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:36 smithi090 ceph-mon[43420]: osdmap e60: 10 total, 9 up, 10 in 2024-04-06T14:02:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1393136635' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:36 smithi134 ceph-mon[47549]: osdmap e60: 10 total, 9 up, 10 in 2024-04-06T14:02:36.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:37 smithi105 ceph-mon[46969]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:37 smithi090 ceph-mon[43420]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:37 smithi134 ceph-mon[47549]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 258 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:38 smithi105 ceph-mon[46969]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:02:38.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:38 smithi090 ceph-mon[43420]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:02:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:38 smithi134 ceph-mon[47549]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-06T14:02:41.449 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:41 smithi090 ceph-mon[43420]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-06T14:02:41.449 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-06T14:02:41.449 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:41 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:41 smithi105 ceph-mon[46969]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-06T14:02:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-06T14:02:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:41 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:41 smithi134 ceph-mon[47549]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-06T14:02:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2024-04-06T14:02:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:41 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:42.477 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:42 smithi090 ceph-mon[43420]: Deploying daemon osd.9 on smithi090 2024-04-06T14:02:42.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:42 smithi105 ceph-mon[46969]: Deploying daemon osd.9 on smithi090 2024-04-06T14:02:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:42 smithi134 ceph-mon[47549]: Deploying daemon osd.9 on smithi090 2024-04-06T14:02:43.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:43 smithi105 ceph-mon[46969]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-06T14:02:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:43 smithi090 ceph-mon[43420]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-06T14:02:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:43 smithi134 ceph-mon[47549]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-06T14:02:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:45 smithi105 ceph-mon[46969]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:02:45.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:45 smithi090 ceph-mon[43420]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:02:45.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:45 smithi134 ceph-mon[47549]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:02:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:46.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.760+0000 7fda537fe700 1 -- 172.21.15.90:0/1560184062 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fda6402ce20 con 0x7fda4c0a1120 2024-04-06T14:02:46.761 INFO:teuthology.orchestra.run.smithi090.stdout:Created osd(s) 9 on host 'smithi090' 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 msgr2=0x7fda4c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 0x7fda4c0a35e0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fda58005950 tx=0x7fda580058e0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 msgr2=0x7fda64196230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fda5400c920 tx=0x7fda5400cce0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 shutdown_connections 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fda4c0a1120 0x7fda4c0a35e0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda641018b0 0x7fda6419d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda640ffd10 0x7fda64195cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 --2- 172.21.15.90:0/1560184062 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda64100f10 0x7fda64196230 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 >> 172.21.15.90:0/1560184062 conn(0x7fda640fb2b0 msgr2=0x7fda64104260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:46.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 shutdown_connections 2024-04-06T14:02:46.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:02:46.763+0000 7fda68d84700 1 -- 172.21.15.90:0/1560184062 wait complete. 2024-04-06T14:02:47.230 DEBUG:teuthology.orchestra.run.smithi090:osd.9> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.9.service 2024-04-06T14:02:47.232 INFO:tasks.cephadm:Deploying osd.10 on smithi105 with /dev/vg_nvme/lv_1... 2024-04-06T14:02:47.232 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:47.256 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.262 INFO:journalctl@ceph.osd.9.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:02:47.473 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:47.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:02:47.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:48.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:48 smithi105 ceph-mon[46969]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-06T14:02:48.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:48 smithi090 ceph-mon[43420]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-06T14:02:48.789 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:02:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:02:48.458+0000 7f2ad4652640 -1 osd.9 0 log_to_monitors true 2024-04-06T14:02:48.827 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-06T14:02:48.827 INFO:teuthology.orchestra.run.smithi105.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-06T14:02:48.827 INFO:teuthology.orchestra.run.smithi105.stderr: stderr: 10+0 records in 2024-04-06T14:02:48.827 INFO:teuthology.orchestra.run.smithi105.stderr:10+0 records out 2024-04-06T14:02:48.828 INFO:teuthology.orchestra.run.smithi105.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126277 s, 830 MB/s 2024-04-06T14:02:48.828 INFO:teuthology.orchestra.run.smithi105.stderr:--> Zapping successful for: 2024-04-06T14:02:48.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:48 smithi134 ceph-mon[47549]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-06T14:02:49.170 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi105:vg_nvme/lv_1 2024-04-06T14:02:49.389 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: osdmap e61: 10 total, 9 up, 10 in 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:02:49.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi090 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 4278M 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:49.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:02:49.526+0000 7f2ac9cb6700 -1 osd.9 0 waiting for initial osdmap 2024-04-06T14:02:49.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:02:49.538+0000 7f2ac5aab700 -1 osd.9 62 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: osdmap e61: 10 total, 9 up, 10 in 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi090 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 4278M 2024-04-06T14:02:49.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:49.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:49.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: osdmap e61: 10 total, 9 up, 10 in 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]: dispatch 2024-04-06T14:02:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi090 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 4278M 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:02:49.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:02:50.450 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- 172.21.15.105:0/967881763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 msgr2=0x7f7c7c101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:50.450 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 --2- 172.21.15.105:0/967881763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c101370 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f7c780099c0 tx=0x7f7c78009cd0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.450 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- 172.21.15.105:0/967881763 shutdown_connections 2024-04-06T14:02:50.450 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 --2- 172.21.15.105:0/967881763 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.450 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 --2- 172.21.15.105:0/967881763 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c0ffd10 0x7f7c7c100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 --2- 172.21.15.105:0/967881763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c101370 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- 172.21.15.105:0/967881763 >> 172.21.15.105:0/967881763 conn(0x7f7c7c0fb2d0 msgr2=0x7f7c7c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- 172.21.15.105:0/967881763 shutdown_connections 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- 172.21.15.105:0/967881763 wait complete. 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 Processor -- start 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.447+0000 7f7c841b8700 1 -- start start 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c0ffd10 0x7f7c7c069b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c19dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c7c19e790 con 0x7f7c7c1018b0 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c7c19e900 con 0x7f7c7c100f10 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c7c19eac0 con 0x7f7c7c0ffd10 2024-04-06T14:02:50.451 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:41310/0 (socket says 172.21.15.105:41310) 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 -- 172.21.15.105:0/1717637480 learned_addr learned my addr 172.21.15.105:0/1717637480 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81f54700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c0ffd10 0x7f7c7c069b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c82755700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c19dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 -- 172.21.15.105:0/1717637480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c0ffd10 msgr2=0x7f7c7c069b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c0ffd10 0x7f7c7c069b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 -- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 msgr2=0x7f7c7c19dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c19dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 -- 172.21.15.105:0/1717637480 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c780095f0 con 0x7f7c7c100f10 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c82755700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c19dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c81753700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f7c780036f0 tx=0x7f7c780047b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c7801d070 con 0x7f7c7c100f10 2024-04-06T14:02:50.452 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c7c19ed40 con 0x7f7c7c100f10 2024-04-06T14:02:50.453 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c78004090 con 0x7f7c7c100f10 2024-04-06T14:02:50.453 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c78022d40 con 0x7f7c7c100f10 2024-04-06T14:02:50.453 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.448+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c7c19f1d0 con 0x7f7c7c100f10 2024-04-06T14:02:50.453 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.449+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7c7c1071e0 con 0x7f7c7c100f10 2024-04-06T14:02:50.454 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.450+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f7c78022470 con 0x7f7c7c100f10 2024-04-06T14:02:50.454 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.451+0000 7f7c72ffd700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 0x7f7c680a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:02:50.454 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.451+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(62..62 src has 1..62) v4 ==== 5981+0+0 (secure 0 0 0) 0x7f7c780cd590 con 0x7f7c7c100f10 2024-04-06T14:02:50.454 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.451+0000 7f7c81f54700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 0x7f7c680a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:02:50.454 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.452+0000 7f7c81f54700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 0x7f7c680a35e0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7c7c100d70 tx=0x7f7c6c00b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:02:50.455 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.455+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7c78097630 con 0x7f7c7c100f10 2024-04-06T14:02:50.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:02:50.662+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f7c7c02cd20 con 0x7f7c680a1120 2024-04-06T14:02:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:50 smithi090 ceph-mon[43420]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:02:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:50 smithi090 ceph-mon[43420]: osdmap e62: 10 total, 9 up, 10 in 2024-04-06T14:02:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:50 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:50 smithi090 ceph-mon[43420]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:50 smithi134 ceph-mon[47549]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:02:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:50 smithi134 ceph-mon[47549]: osdmap e62: 10 total, 9 up, 10 in 2024-04-06T14:02:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:50 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:50 smithi134 ceph-mon[47549]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:50 smithi105 ceph-mon[46969]: from='osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526]' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi090", "root=default"]}]': finished 2024-04-06T14:02:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:50 smithi105 ceph-mon[46969]: osdmap e62: 10 total, 9 up, 10 in 2024-04-06T14:02:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:50 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:50 smithi105 ceph-mon[46969]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 259 MiB used, 804 GiB / 805 GiB avail 2024-04-06T14:02:51.715 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] boot 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: osdmap e63: 10 total, 10 up, 10 in 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:51.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:51.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:02:51.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:02:51.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] boot 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: osdmap e63: 10 total, 10 up, 10 in 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:51.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: osd.9 [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] boot 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: osdmap e63: 10 total, 10 up, 10 in 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:02:51.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:02:51.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:02:51.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:52.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: osdmap e64: 10 total, 10 up, 10 in 2024-04-06T14:02:52.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:52.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/372689307' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]': finished 2024-04-06T14:02:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: osdmap e65: 11 total, 10 up, 11 in 2024-04-06T14:02:52.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: osdmap e64: 10 total, 10 up, 10 in 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/372689307' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]': finished 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: osdmap e65: 11 total, 10 up, 11 in 2024-04-06T14:02:52.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: osdmap e64: 10 total, 10 up, 10 in 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/372689307' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]: dispatch 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "86cce4f5-4741-4f2b-87e5-0af0d083ae8e"}]': finished 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: osdmap e65: 11 total, 10 up, 11 in 2024-04-06T14:02:52.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:02:53.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:53 smithi090 ceph-mon[43420]: from='client.? 172.21.15.105:0/1908565284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:53.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:53 smithi134 ceph-mon[47549]: from='client.? 172.21.15.105:0/1908565284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:53 smithi105 ceph-mon[46969]: from='client.? 172.21.15.105:0/1908565284' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:02:54.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:54 smithi134 ceph-mon[47549]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:54.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:54 smithi105 ceph-mon[46969]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:54 smithi090 ceph-mon[43420]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:57 smithi105 ceph-mon[46969]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:57.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:57 smithi090 ceph-mon[43420]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:57.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:57 smithi134 ceph-mon[47549]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:58.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:58 smithi105 ceph-mon[46969]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:58.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-06T14:02:58.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:58.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:58 smithi090 ceph-mon[43420]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:58.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-06T14:02:58.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:58 smithi134 ceph-mon[47549]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:02:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2024-04-06T14:02:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:02:59.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:02:59 smithi105 ceph-mon[46969]: Deploying daemon osd.10 on smithi105 2024-04-06T14:02:59.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:02:59 smithi090 ceph-mon[43420]: Deploying daemon osd.10 on smithi105 2024-04-06T14:02:59.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:02:59 smithi134 ceph-mon[47549]: Deploying daemon osd.10 on smithi105 2024-04-06T14:03:00.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:00 smithi105 ceph-mon[46969]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:00 smithi090 ceph-mon[43420]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:00.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:00 smithi134 ceph-mon[47549]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:03.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:03 smithi105 ceph-mon[46969]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:03 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:03 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:03 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:03 smithi090 ceph-mon[43420]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:03 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:03 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:03 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:03.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:03 smithi134 ceph-mon[47549]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:03.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:03 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:03.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:03 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:03.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:03 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:04.519 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.518+0000 7f7c72ffd700 1 -- 172.21.15.105:0/1717637480 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7f7c7c02cd20 con 0x7f7c680a1120 2024-04-06T14:03:04.519 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 10 on host 'smithi105' 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 msgr2=0x7f7c680a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 0x7f7c680a35e0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f7c7c100d70 tx=0x7f7c6c00b410 comp rx=0 tx=0).stop 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 msgr2=0x7f7c7c19ba00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f7c780036f0 tx=0x7f7c780047b0 comp rx=0 tx=0).stop 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 shutdown_connections 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c680a1120 0x7f7c680a35e0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c7c1018b0 0x7f7c7c19dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c7c0ffd10 0x7f7c7c069b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:04.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 --2- 172.21.15.105:0/1717637480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c7c100f10 0x7f7c7c19ba00 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:04.522 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 >> 172.21.15.105:0/1717637480 conn(0x7f7c7c0fb2d0 msgr2=0x7f7c7c104320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:04.522 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 shutdown_connections 2024-04-06T14:03:04.522 INFO:teuthology.orchestra.run.smithi105.stderr:2024-04-06T14:03:04.521+0000 7f7c841b8700 1 -- 172.21.15.105:0/1717637480 wait complete. 2024-04-06T14:03:04.914 DEBUG:teuthology.orchestra.run.smithi105:osd.10> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.10.service 2024-04-06T14:03:04.917 INFO:tasks.cephadm:Deploying osd.11 on smithi134 with /dev/vg_nvme/lv_1... 2024-04-06T14:03:04.917 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-06T14:03:04.949 INFO:journalctl@ceph.osd.10.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:05.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:05.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:05.099 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.100 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:05.100 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.100 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.100 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:05.155 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:03:05.235 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.236 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10+0 records in 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr:10+0 records out 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0156261 s, 671 MB/s 2024-04-06T14:03:06.385 INFO:teuthology.orchestra.run.smithi134.stderr:--> Zapping successful for: 2024-04-06T14:03:06.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:03:06 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:03:06.121+0000 7f1ee1ee8640 -1 osd.10 0 log_to_monitors true 2024-04-06T14:03:06.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:06 smithi105 ceph-mon[46969]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:06 smithi105 ceph-mon[46969]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:06 smithi090 ceph-mon[43420]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:06 smithi090 ceph-mon[43420]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.627 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:06 smithi134 ceph-mon[47549]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.627 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:06 smithi134 ceph-mon[47549]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2024-04-06T14:03:06.805 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi134:vg_nvme/lv_1 2024-04-06T14:03:07.026 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: osdmap e66: 11 total, 10 up, 11 in 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:07.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:07 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:03:07.198+0000 7f1ed754c700 -1 osd.10 0 waiting for initial osdmap 2024-04-06T14:03:07.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:03:07.211+0000 7f1ed3341700 -1 osd.10 67 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: osdmap e66: 11 total, 10 up, 11 in 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:07.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:07 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: osdmap e66: 11 total, 10 up, 11 in 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:07.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:07.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:07 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:07.871 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 -- 172.21.15.134:0/895119261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1020b0 msgr2=0x7f573c102510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:07.871 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 --2- 172.21.15.134:0/895119261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1020b0 0x7f573c102510 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f5724009990 tx=0x7f5724009ca0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 -- 172.21.15.134:0/895119261 shutdown_connections 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 --2- 172.21.15.134:0/895119261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c102a50 0x7f573c109300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 --2- 172.21.15.134:0/895119261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c0ff4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 --2- 172.21.15.134:0/895119261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1020b0 0x7f573c102510 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 -- 172.21.15.134:0/895119261 >> 172.21.15.134:0/895119261 conn(0x7f573c0faa50 msgr2=0x7f573c0fce70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 -- 172.21.15.134:0/895119261 shutdown_connections 2024-04-06T14:03:07.872 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.870+0000 7f5741158700 1 -- 172.21.15.134:0/895119261 wait complete. 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.871+0000 7f5741158700 1 Processor -- start 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 -- start start 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c196d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 0x7f573c197270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:07.874 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19e830 con 0x7f573c0ff0d0 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19e970 con 0x7f573c1020b0 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f5741158700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19eb30 con 0x7f573c102a50 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:56734/0 (socket says 172.21.15.134:56734) 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 -- 172.21.15.134:0/3887884164 learned_addr learned my addr 172.21.15.134:0/3887884164 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573a59c700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 0x7f573c197270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573ad9d700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c196d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 -- 172.21.15.134:0/3887884164 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 msgr2=0x7f573c197270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 0x7f573c197270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 -- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 msgr2=0x7f573c196d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c196d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 -- 172.21.15.134:0/3887884164 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57240095f0 con 0x7f573c102a50 2024-04-06T14:03:07.875 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573ad9d700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c196d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573a59c700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 0x7f573c197270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.872+0000 7f573b59e700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f573400ef30 tx=0x7f573400c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.873+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5734018070 con 0x7f573c102a50 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.873+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f573400f040 con 0x7f573c102a50 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.873+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f573c19edb0 con 0x7f573c102a50 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.873+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f573400cdb0 con 0x7f573c102a50 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.873+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f573c19f1a0 con 0x7f573c102a50 2024-04-06T14:03:07.876 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.874+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f573c108390 con 0x7f573c102a50 2024-04-06T14:03:07.877 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.875+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f5734003a00 con 0x7f573c102a50 2024-04-06T14:03:07.877 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.875+0000 7f5733fff700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 0x7f57280a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:07.877 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.875+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 6727+0+0 (secure 0 0 0) 0x7f57340cc800 con 0x7f573c102a50 2024-04-06T14:03:07.877 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.876+0000 7f573ad9d700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 0x7f57280a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:07.877 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.876+0000 7f573ad9d700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 0x7f57280a35e0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f572c005fd0 tx=0x7f572c005dd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:07.879 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:07.879+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5734096650 con 0x7f573c102a50 2024-04-06T14:03:08.080 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:08.080+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f573c02cdc0 con 0x7f57280a1120 2024-04-06T14:03:08.457 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi105 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 4278M 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: osdmap e67: 11 total, 10 up, 11 in 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] boot 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: osdmap e68: 11 total, 11 up, 11 in 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:03:08.458 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:08 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi105 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 4278M 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: osdmap e67: 11 total, 10 up, 11 in 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] boot 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: osdmap e68: 11 total, 11 up, 11 in 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:03:08.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:08 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi105 2024-04-06T14:03:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 4278M 2024-04-06T14:03:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-04-06T14:03:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: osdmap e67: 11 total, 10 up, 11 in 2024-04-06T14:03:08.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: osd.10 [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] boot 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: osdmap e68: 11 total, 11 up, 11 in 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-06T14:03:08.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:08 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:09 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:03:09.568 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:09 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:03:09.568 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:09 smithi134 ceph-mon[47549]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:09.568 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:09 smithi134 ceph-mon[47549]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:09.568 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:09 smithi134 ceph-mon[47549]: osdmap e69: 11 total, 11 up, 11 in 2024-04-06T14:03:09.569 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:09 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:03:09.569 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:09 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:03:09.569 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:09 smithi090 ceph-mon[43420]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:09.569 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:09 smithi090 ceph-mon[43420]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:09.569 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:09 smithi090 ceph-mon[43420]: osdmap e69: 11 total, 11 up, 11 in 2024-04-06T14:03:09.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:09 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:03:09.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:09 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:03:09.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:09 smithi105 ceph-mon[46969]: from='client.24370 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi134:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:09.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:09 smithi105 ceph-mon[46969]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 286 MiB used, 894 GiB / 894 GiB avail 2024-04-06T14:03:09.570 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:09 smithi105 ceph-mon[46969]: osdmap e69: 11 total, 11 up, 11 in 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: osdmap e70: 11 total, 11 up, 11 in 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/367867286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]': finished 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: osdmap e71: 12 total, 11 up, 12 in 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:10.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:10 smithi105 ceph-mon[46969]: pgmap v184: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: osdmap e70: 11 total, 11 up, 11 in 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/367867286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]': finished 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: osdmap e71: 12 total, 11 up, 12 in 2024-04-06T14:03:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:10.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:10 smithi090 ceph-mon[43420]: pgmap v184: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: osdmap e70: 11 total, 11 up, 11 in 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/367867286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]: dispatch 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1942c563-7632-4c97-b0a2-7f079c532b07"}]': finished 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: osdmap e71: 12 total, 11 up, 12 in 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:10 smithi134 ceph-mon[47549]: pgmap v184: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:11.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:11 smithi105 ceph-mon[46969]: from='client.? 172.21.15.134:0/636227805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:03:11.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:11 smithi105 ceph-mon[46969]: osdmap e72: 12 total, 11 up, 12 in 2024-04-06T14:03:11.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:11 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:11.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.134:0/636227805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:03:11.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:11 smithi090 ceph-mon[43420]: osdmap e72: 12 total, 11 up, 12 in 2024-04-06T14:03:11.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:11 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:11 smithi134 ceph-mon[47549]: from='client.? 172.21.15.134:0/636227805' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-06T14:03:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:11 smithi134 ceph-mon[47549]: osdmap e72: 12 total, 11 up, 12 in 2024-04-06T14:03:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:11 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:12 smithi134 ceph-mon[47549]: osdmap e73: 12 total, 11 up, 12 in 2024-04-06T14:03:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:12 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:12 smithi134 ceph-mon[47549]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:13.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:12 smithi105 ceph-mon[46969]: osdmap e73: 12 total, 11 up, 12 in 2024-04-06T14:03:13.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:12 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:13.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:12 smithi105 ceph-mon[46969]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:12 smithi090 ceph-mon[43420]: osdmap e73: 12 total, 11 up, 12 in 2024-04-06T14:03:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:12 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:12 smithi090 ceph-mon[43420]: pgmap v187: 1 pgs: 1 peering; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:15.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:15 smithi105 ceph-mon[46969]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-06T14:03:15.537 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:15 smithi134 ceph-mon[47549]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-06T14:03:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:15 smithi090 ceph-mon[43420]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-06T14:03:16.259 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-06T14:03:16.259 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:16 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-06T14:03:16.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:16 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2024-04-06T14:03:16.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:16 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:17.295 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:17 smithi134 ceph-mon[47549]: Deploying daemon osd.11 on smithi134 2024-04-06T14:03:17.296 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:17 smithi134 ceph-mon[47549]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-06T14:03:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:17 smithi105 ceph-mon[46969]: Deploying daemon osd.11 on smithi134 2024-04-06T14:03:17.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:17 smithi105 ceph-mon[46969]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-06T14:03:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:17 smithi090 ceph-mon[43420]: Deploying daemon osd.11 on smithi134 2024-04-06T14:03:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:17 smithi090 ceph-mon[43420]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-04-06T14:03:18.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:18 smithi105 ceph-mon[46969]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:03:18.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:18 smithi090 ceph-mon[43420]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:03:18.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:18 smithi134 ceph-mon[47549]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-06T14:03:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:20 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:20.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:20 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:20.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:20 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:21.282 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.279+0000 7f5733fff700 1 -- 172.21.15.134:0/3887884164 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7f573c02cdc0 con 0x7f57280a1120 2024-04-06T14:03:21.283 INFO:teuthology.orchestra.run.smithi134.stdout:Created osd(s) 11 on host 'smithi134' 2024-04-06T14:03:21.283 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 msgr2=0x7f57280a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 0x7f57280a35e0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f572c005fd0 tx=0x7f572c005dd0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 msgr2=0x7f573c19e260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f573400ef30 tx=0x7f573400c3a0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 shutdown_connections 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f57280a1120 0x7f57280a35e0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c102a50 0x7f573c19e260 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c0ff0d0 0x7f573c196d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 --2- 172.21.15.134:0/3887884164 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1020b0 0x7f573c197270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 >> 172.21.15.134:0/3887884164 conn(0x7f573c0faa50 msgr2=0x7f573c1054d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 shutdown_connections 2024-04-06T14:03:21.284 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:21.281+0000 7f5741158700 1 -- 172.21.15.134:0/3887884164 wait complete. 2024-04-06T14:03:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:03:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:21 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.542 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:21 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.633 DEBUG:teuthology.orchestra.run.smithi134:osd.11> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.11.service 2024-04-06T14:03:21.635 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2024-04-06T14:03:21.635 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-06T14:03:21.668 INFO:journalctl@ceph.osd.11.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:21 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:21.872 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:03:22.274 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:22 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:22.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:22 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:22.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:22 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:22.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 -- 172.21.15.90:0/2486409492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c8100f20 msgr2=0x7fd8c8101380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:22.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2486409492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c8100f20 0x7fd8c8101380 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7fd8c40099c0 tx=0x7fd8c4009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 -- 172.21.15.90:0/2486409492 shutdown_connections 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2486409492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 0x7fd8c8108170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2486409492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c80ffd20 0x7fd8c8100140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2486409492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c8100f20 0x7fd8c8101380 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.880+0000 7fd8d0632700 1 -- 172.21.15.90:0/2486409492 >> 172.21.15.90:0/2486409492 conn(0x7fd8c80fb2a0 msgr2=0x7fd8c80fd700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.881+0000 7fd8d0632700 1 -- 172.21.15.90:0/2486409492 shutdown_connections 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.881+0000 7fd8d0632700 1 -- 172.21.15.90:0/2486409492 wait complete. 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.881+0000 7fd8d0632700 1 Processor -- start 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.881+0000 7fd8d0632700 1 -- start start 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c80ffd20 0x7fd8c8195c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 0x7fd8c819d190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:22.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd8c819d760 con 0x7fd8c80ffd20 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd8c819d8a0 con 0x7fd8c81018c0 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd8c819da30 con 0x7fd8c8100f20 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42490/0 (socket says 172.21.15.90:42490) 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 -- 172.21.15.90:0/2243332129 learned_addr learned my addr 172.21.15.90:0/2243332129 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cebcf700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 0x7fd8c819d190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8ce3ce700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c80ffd20 0x7fd8c8195c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 -- 172.21.15.90:0/2243332129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 msgr2=0x7fd8c819d190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 0x7fd8c819d190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 -- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c80ffd20 msgr2=0x7fd8c8195c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c80ffd20 0x7fd8c8195c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 -- 172.21.15.90:0/2243332129 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd8c40095f0 con 0x7fd8c8100f20 2024-04-06T14:03:22.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8cdbcd700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fd8c4009f50 tx=0x7fd8c4005af0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:22.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd8c401d070 con 0x7fd8c8100f20 2024-04-06T14:03:22.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd8c8069590 con 0x7fd8c8100f20 2024-04-06T14:03:22.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.882+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd8c4004150 con 0x7fd8c8100f20 2024-04-06T14:03:22.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.883+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd8c40053b0 con 0x7fd8c8100f20 2024-04-06T14:03:22.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.883+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd8c8069a80 con 0x7fd8c8100f20 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.884+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd8c804ea80 con 0x7fd8c8100f20 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.885+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fd8c4005510 con 0x7fd8c8100f20 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.885+0000 7fd8bb7fe700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 0x7fd8b40a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.885+0000 7fd8ce3ce700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 0x7fd8b40a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.885+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(73..73 src has 1..73) v4 ==== 6809+0+0 (secure 0 0 0) 0x7fd8c40cd270 con 0x7fd8c8100f20 2024-04-06T14:03:22.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.886+0000 7fd8ce3ce700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 0x7fd8b40a35e0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fd8bc005950 tx=0x7fd8bc0058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:22.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:22.889+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd8c4097050 con 0x7fd8c8100f20 2024-04-06T14:03:23.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.083+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7fd8c810b570 con 0x7fd8c8100f20 2024-04-06T14:03:23.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.084+0000 7fd8bb7fe700 1 -- 172.21.15.90:0/2243332129 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v73) v1 ==== 74+0+133 (secure 0 0 0) 0x7fd8c409a670 con 0x7fd8c8100f20 2024-04-06T14:03:23.085 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:23.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 msgr2=0x7fd8b40a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:23.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 0x7fd8b40a35e0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fd8bc005950 tx=0x7fd8bc0058e0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 msgr2=0x7fd8c8196160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fd8c4009f50 tx=0x7fd8c4005af0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 shutdown_connections 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd8b40a1120 0x7fd8b40a35e0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8c81018c0 0x7fd8c819d190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8c80ffd20 0x7fd8c8195c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 --2- 172.21.15.90:0/2243332129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8c8100f20 0x7fd8c8196160 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 >> 172.21.15.90:0/2243332129 conn(0x7fd8c80fb2a0 msgr2=0x7fd8c8104340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 shutdown_connections 2024-04-06T14:03:23.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:23.086+0000 7fd8d0632700 1 -- 172.21.15.90:0/2243332129 wait complete. 2024-04-06T14:03:23.384 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:23 smithi090 ceph-mon[43420]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:03:23.384 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:23 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2243332129' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:23.384 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:23 smithi090 ceph-mon[43420]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:23.384 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:23 smithi090 ceph-mon[43420]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:23.460 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":73,"num_osds":12,"num_up_osds":11,"osd_up_since":1712412187,"num_in_osds":12,"osd_in_since":1712412189,"num_remapped_pgs":0} 2024-04-06T14:03:23.535 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:23 smithi134 ceph-mon[47549]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:03:23.535 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:23 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2243332129' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:23.535 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:23 smithi134 ceph-mon[47549]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:23.536 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:23 smithi134 ceph-mon[47549]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:23.536 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:03:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:03:23.164+0000 7f4184ca5640 -1 osd.11 0 log_to_monitors true 2024-04-06T14:03:23.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:23 smithi105 ceph-mon[46969]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-04-06T14:03:23.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:23 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2243332129' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:23.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:23 smithi105 ceph-mon[46969]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:23.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:23 smithi105 ceph-mon[46969]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2024-04-06T14:03:24.461 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-06T14:03:24.686 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: osdmap e74: 12 total, 11 up, 12 in 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: Detected new or changed devices on smithi134 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 4278M 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:24 smithi105 ceph-mon[46969]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-06T14:03:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: osdmap e74: 12 total, 11 up, 12 in 2024-04-06T14:03:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: Detected new or changed devices on smithi134 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 4278M 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:24 smithi090 ceph-mon[43420]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: osdmap e74: 12 total, 11 up, 12 in 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]: dispatch 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: Detected new or changed devices on smithi134 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 4278M 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-mon[47549]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 314 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:25.393 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:03:24.965+0000 7f417a309700 -1 osd.11 0 waiting for initial osdmap 2024-04-06T14:03:25.393 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:03:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:03:24.981+0000 7f41720f6700 -1 osd.11 75 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/1479665628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 msgr2=0x7fdb54100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/1479665628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54100130 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fdb480099c0 tx=0x7fdb48009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/1479665628 shutdown_connections 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/1479665628 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/1479665628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54100130 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/1479665628 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 0x7fdb54101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/1479665628 >> 172.21.15.90:0/1479665628 conn(0x7fdb540fb2b0 msgr2=0x7fdb540fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/1479665628 shutdown_connections 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/1479665628 wait complete. 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 Processor -- start 2024-04-06T14:03:25.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.722+0000 7fdb5a9f3700 1 -- start start 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 0x7fdb54102500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54102a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb541031e0 con 0x7fdb540ffd10 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb54103350 con 0x7fdb541018b0 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb5a9f3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb541a0de0 con 0x7fdb54100f10 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56562/0 (socket says 172.21.15.90:56562) 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 -- 172.21.15.90:0/685133643 learned_addr learned my addr 172.21.15.90:0/685133643 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb537fe700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 0x7fdb54102500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb58f90700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54102a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:25.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 -- 172.21.15.90:0/685133643 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 msgr2=0x7fdb54102500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 0x7fdb54102500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 -- 172.21.15.90:0/685133643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 msgr2=0x7fdb54102a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54102a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 -- 172.21.15.90:0/685133643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb480095f0 con 0x7fdb540ffd10 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb58f90700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54102a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb53fff700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fdb48003820 tx=0x7fdb48005ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.723+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb4801d070 con 0x7fdb540ffd10 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.724+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb541a1010 con 0x7fdb540ffd10 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.724+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdb48004110 con 0x7fdb540ffd10 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.724+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb48022e40 con 0x7fdb540ffd10 2024-04-06T14:03:25.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.724+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb541a1500 con 0x7fdb540ffd10 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.725+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb5404ea80 con 0x7fdb540ffd10 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.726+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fdb48022570 con 0x7fdb540ffd10 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.726+0000 7fdb517fa700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 0x7fdb3c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.726+0000 7fdb537fe700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 0x7fdb3c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.726+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(75..75 src has 1..75) v4 ==== 6847+0+0 (secure 0 0 0) 0x7fdb480cd720 con 0x7fdb540ffd10 2024-04-06T14:03:25.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.727+0000 7fdb537fe700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 0x7fdb3c0a35e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fdb44005950 tx=0x7fdb440058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:25.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.731+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fdb48097500 con 0x7fdb540ffd10 2024-04-06T14:03:25.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:25 smithi105 ceph-mon[46969]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:03:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:25 smithi105 ceph-mon[46969]: osdmap e75: 12 total, 11 up, 12 in 2024-04-06T14:03:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:25 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:25 smithi105 ceph-mon[46969]: from='osd.11 ' entity='osd.11' 2024-04-06T14:03:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:25 smithi090 ceph-mon[43420]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:03:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:25 smithi090 ceph-mon[43420]: osdmap e75: 12 total, 11 up, 12 in 2024-04-06T14:03:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:25 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:25 smithi090 ceph-mon[43420]: from='osd.11 ' entity='osd.11' 2024-04-06T14:03:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:25 smithi134 ceph-mon[47549]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi134", "root=default"]}]': finished 2024-04-06T14:03:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:25 smithi134 ceph-mon[47549]: osdmap e75: 12 total, 11 up, 12 in 2024-04-06T14:03:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:25 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:25 smithi134 ceph-mon[47549]: from='osd.11 ' entity='osd.11' 2024-04-06T14:03:25.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.927+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7fdb5410b510 con 0x7fdb540ffd10 2024-04-06T14:03:25.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.927+0000 7fdb517fa700 1 -- 172.21.15.90:0/685133643 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v75) v1 ==== 74+0+133 (secure 0 0 0) 0x7fdb4809ab20 con 0x7fdb540ffd10 2024-04-06T14:03:25.928 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:25.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 msgr2=0x7fdb3c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:25.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 0x7fdb3c0a35e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fdb44005950 tx=0x7fdb440058e0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 msgr2=0x7fdb54103e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fdb48003820 tx=0x7fdb48005ad0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 shutdown_connections 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdb3c0a1120 0x7fdb3c0a35e0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdb541018b0 0x7fdb54102a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.929+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdb540ffd10 0x7fdb54103e60 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.930+0000 7fdb5a9f3700 1 --2- 172.21.15.90:0/685133643 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdb54100f10 0x7fdb54102500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.930+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 >> 172.21.15.90:0/685133643 conn(0x7fdb540fb2b0 msgr2=0x7fdb54068760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.930+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 shutdown_connections 2024-04-06T14:03:25.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:25.930+0000 7fdb5a9f3700 1 -- 172.21.15.90:0/685133643 wait complete. 2024-04-06T14:03:26.318 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":75,"num_osds":12,"num_up_osds":11,"osd_up_since":1712412187,"num_in_osds":12,"osd_in_since":1712412189,"num_remapped_pgs":0} 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: purged_snaps scrub starts 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: purged_snaps scrub ok 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/685133643' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] boot 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: osdmap e76: 12 total, 12 up, 12 in 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:26 smithi105 ceph-mon[46969]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 315 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: purged_snaps scrub starts 2024-04-06T14:03:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: purged_snaps scrub ok 2024-04-06T14:03:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/685133643' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] boot 2024-04-06T14:03:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: osdmap e76: 12 total, 12 up, 12 in 2024-04-06T14:03:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:26 smithi090 ceph-mon[43420]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 315 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: purged_snaps scrub starts 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: purged_snaps scrub ok 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/685133643' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: osd.11 [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] boot 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: osdmap e76: 12 total, 12 up, 12 in 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:03:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:26 smithi134 ceph-mon[47549]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 315 MiB used, 983 GiB / 983 GiB avail 2024-04-06T14:03:27.319 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-06T14:03:27.547 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:03:28.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:27 smithi105 ceph-mon[46969]: osdmap e77: 12 total, 12 up, 12 in 2024-04-06T14:03:28.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:27 smithi090 ceph-mon[43420]: osdmap e77: 12 total, 12 up, 12 in 2024-04-06T14:03:28.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:27 smithi134 ceph-mon[47549]: osdmap e77: 12 total, 12 up, 12 in 2024-04-06T14:03:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/3023768358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f40ff710 msgr2=0x7ff2f40ffb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/3023768358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f40ff710 0x7ff2f40ffb30 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7ff2e8009990 tx=0x7ff2e8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/3023768358 shutdown_connections 2024-04-06T14:03:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/3023768358 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f4100a30 0x7ff2f4100eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/3023768358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f40ff710 0x7ff2f40ffb30 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/3023768358 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 0x7ff2f41004f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/3023768358 >> 172.21.15.90:0/3023768358 conn(0x7ff2f40fb2b0 msgr2=0x7ff2f40fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:28.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/3023768358 shutdown_connections 2024-04-06T14:03:28.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.522+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/3023768358 wait complete. 2024-04-06T14:03:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 Processor -- start 2024-04-06T14:03:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 -- start start 2024-04-06T14:03:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f40ff710 0x7ff2f4198f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 0x7ff2f4199470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2f41a0a70 con 0x7ff2f4100a30 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2f41a0bb0 con 0x7ff2f40ff710 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fc1c7700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2f41a0d40 con 0x7ff2f4100070 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fa764700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fa764700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56572/0 (socket says 172.21.15.90:56572) 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2fa764700 1 -- 172.21.15.90:0/692504879 learned_addr learned my addr 172.21.15.90:0/692504879 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2f9f63700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f40ff710 0x7ff2f4198f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.523+0000 7ff2f9762700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 0x7ff2f4199470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 -- 172.21.15.90:0/692504879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 msgr2=0x7ff2f4199470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 0x7ff2f4199470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 -- 172.21.15.90:0/692504879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f40ff710 msgr2=0x7ff2f4198f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f40ff710 0x7ff2f4198f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 -- 172.21.15.90:0/692504879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff2e80095f0 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fa764700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7ff2ec00add0 tx=0x7ff2ec00c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff2ec00f070 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff2ec007b30 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff2f41a0fc0 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff2ec005280 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.524+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff2f41a14e0 con 0x7ff2f4100a30 2024-04-06T14:03:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.525+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff2f410a4f0 con 0x7ff2f4100a30 2024-04-06T14:03:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.526+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7ff2ec00c890 con 0x7ff2f4100a30 2024-04-06T14:03:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.527+0000 7ff2e6ffd700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 0x7ff2e00a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.527+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7ff2ec0cbe80 con 0x7ff2f4100a30 2024-04-06T14:03:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.527+0000 7ff2f9f63700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 0x7ff2e00a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.527+0000 7ff2f9f63700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 0x7ff2e00a3660 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7ff2e800b5c0 tx=0x7ff2e8003790 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:28.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.530+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff2ec095b40 con 0x7ff2f4100a30 2024-04-06T14:03:28.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.723+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7ff2f410e7f0 con 0x7ff2f4100a30 2024-04-06T14:03:28.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.724+0000 7ff2e6ffd700 1 -- 172.21.15.90:0/692504879 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v77) v1 ==== 74+0+133 (secure 0 0 0) 0x7ff2ec099160 con 0x7ff2f4100a30 2024-04-06T14:03:28.724 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:28.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.725+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 msgr2=0x7ff2e00a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:28.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.725+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 0x7ff2e00a3660 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7ff2e800b5c0 tx=0x7ff2e8003790 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 msgr2=0x7ff2f41a04a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7ff2ec00add0 tx=0x7ff2ec00c330 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 shutdown_connections 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff2e00a11a0 0x7ff2e00a3660 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff2f4100a30 0x7ff2f41a04a0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff2f40ff710 0x7ff2f4198f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 --2- 172.21.15.90:0/692504879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff2f4100070 0x7ff2f4199470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 >> 172.21.15.90:0/692504879 conn(0x7ff2f40fb2b0 msgr2=0x7ff2f40fcdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 shutdown_connections 2024-04-06T14:03:28.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:28.726+0000 7ff2fc1c7700 1 -- 172.21.15.90:0/692504879 wait complete. 2024-04-06T14:03:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:28 smithi090 ceph-mon[43420]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:28 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/692504879' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:29.126 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":77,"num_osds":12,"num_up_osds":12,"osd_up_since":1712412205,"num_in_osds":12,"osd_in_since":1712412189,"num_remapped_pgs":0} 2024-04-06T14:03:29.127 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-06T14:03:29.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:28 smithi105 ceph-mon[46969]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:29.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:28 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/692504879' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:29.346 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:03:29.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:28 smithi134 ceph-mon[47549]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:29.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:28 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/692504879' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-06T14:03:30.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- 172.21.15.90:0/969027773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac100990 msgr2=0x7f76ac10caa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:30.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 --2- 172.21.15.90:0/969027773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac100990 0x7f76ac10caa0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f76a400b0d0 tx=0x7f76a400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- 172.21.15.90:0/969027773 shutdown_connections 2024-04-06T14:03:30.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 --2- 172.21.15.90:0/969027773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac100990 0x7f76ac10caa0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 --2- 172.21.15.90:0/969027773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac105e50 0x7f76ac106230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 --2- 172.21.15.90:0/969027773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 0x7f76ac100450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- 172.21.15.90:0/969027773 >> 172.21.15.90:0/969027773 conn(0x7f76ac076a50 msgr2=0x7f76ac076e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- 172.21.15.90:0/969027773 shutdown_connections 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- 172.21.15.90:0/969027773 wait complete. 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 Processor -- start 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.314+0000 7f76b386a700 1 -- start start 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 0x7f76ac1950e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac100990 0x7f76ac195620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76ac199f90 con 0x7f76ac105e50 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76ac19a100 con 0x7f76ac100990 2024-04-06T14:03:30.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76ac19a2c0 con 0x7f76ac0fffd0 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56586/0 (socket says 172.21.15.90:56586) 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 -- 172.21.15.90:0/118382214 learned_addr learned my addr 172.21.15.90:0/118382214 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1606700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 0x7f76ac1950e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b0e05700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac100990 0x7f76ac195620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 -- 172.21.15.90:0/118382214 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 msgr2=0x7f76ac1950e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 0x7f76ac1950e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 -- 172.21.15.90:0/118382214 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac100990 msgr2=0x7f76ac195620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac100990 0x7f76ac195620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 -- 172.21.15.90:0/118382214 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76a4009d00 con 0x7f76ac105e50 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b1e07700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f76a4009440 tx=0x7f76a4009470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a400e040 con 0x7f76ac105e50 2024-04-06T14:03:30.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f76a401d940 con 0x7f76ac105e50 2024-04-06T14:03:30.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76ac19a540 con 0x7f76ac105e50 2024-04-06T14:03:30.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76a401b890 con 0x7f76ac105e50 2024-04-06T14:03:30.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.315+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76ac1a1e00 con 0x7f76ac105e50 2024-04-06T14:03:30.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.316+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f76ac1080f0 con 0x7f76ac105e50 2024-04-06T14:03:30.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.317+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f76a401d480 con 0x7f76ac105e50 2024-04-06T14:03:30.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.318+0000 7f76a27fc700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 0x7f76980a57f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:30.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.318+0000 7f76b1606700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 0x7f76980a57f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:30.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.318+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f76a40ce360 con 0x7f76ac105e50 2024-04-06T14:03:30.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.318+0000 7f76b1606700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 0x7f76980a57f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f76a8005950 tx=0x7f76a800a300 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.321+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f76a4098020 con 0x7f76ac105e50 2024-04-06T14:03:30.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.514+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f76ac04eab0 con 0x7f76ac105e50 2024-04-06T14:03:30.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.515+0000 7f76a27fc700 1 -- 172.21.15.90:0/118382214 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v77) v1 ==== 74+0+18411 (secure 0 0 0) 0x7f76a409b640 con 0x7f76ac105e50 2024-04-06T14:03:30.516 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:30.516 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":77,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:03:26.970706+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 msgr2=0x7f76980a57f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 0x7f76980a57f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f76a8005950 tx=0x7f76a800a300 comp rx=0 tx=0).stop 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 msgr2=0x7f76ac199980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f76a4009440 tx=0x7f76a4009470 comp rx=0 tx=0).stop 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 shutdown_connections 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76980a3330 0x7f76980a57f0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76ac105e50 0x7f76ac199980 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76ac0fffd0 0x7f76ac1950e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 --2- 172.21.15.90:0/118382214 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76ac100990 0x7f76ac195620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:30.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 >> 172.21.15.90:0/118382214 conn(0x7f76ac076a50 msgr2=0x7f76ac0fee60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:30.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 shutdown_connections 2024-04-06T14:03:30.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:30.517+0000 7f76b386a700 1 -- 172.21.15.90:0/118382214 wait complete. 2024-04-06T14:03:30.927 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:03:30.928 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-06T14:03:31.148 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:03:31.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:31 smithi105 ceph-mon[46969]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:31.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/118382214' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:31 smithi090 ceph-mon[43420]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/118382214' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:31.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:31 smithi134 ceph-mon[47549]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:31.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/118382214' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 -- 172.21.15.90:0/3620546102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 msgr2=0x7f68cc1071d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 --2- 172.21.15.90:0/3620546102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1071d0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f68bc0099c0 tx=0x7f68bc009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 -- 172.21.15.90:0/3620546102 shutdown_connections 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 --2- 172.21.15.90:0/3620546102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc0699e0 0x7f68cc10dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 --2- 172.21.15.90:0/3620546102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1071d0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 --2- 172.21.15.90:0/3620546102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc069020 0x7f68cc0694a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 -- 172.21.15.90:0/3620546102 >> 172.21.15.90:0/3620546102 conn(0x7f68cc076c80 msgr2=0x7f68cc077090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 -- 172.21.15.90:0/3620546102 shutdown_connections 2024-04-06T14:03:32.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 -- 172.21.15.90:0/3620546102 wait complete. 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.125+0000 7f68d1deb700 1 Processor -- start 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 -- start start 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 0x7f68cc0ffd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc0699e0 0x7f68cc1002b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68cc1a5460 con 0x7f68cc106df0 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68cc1a55d0 con 0x7f68cc0699e0 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68d1deb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68cc1a5790 con 0x7f68cc069020 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56602/0 (socket says 172.21.15.90:56602) 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 -- 172.21.15.90:0/1362458938 learned_addr learned my addr 172.21.15.90:0/1362458938 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:32.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68caffd700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc0699e0 0x7f68cc1002b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cb7fe700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 0x7f68cc0ffd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 -- 172.21.15.90:0/1362458938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 msgr2=0x7f68cc0ffd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 0x7f68cc0ffd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 -- 172.21.15.90:0/1362458938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc0699e0 msgr2=0x7f68cc1002b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc0699e0 0x7f68cc1002b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 -- 172.21.15.90:0/1362458938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68bc0095f0 con 0x7f68cc106df0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cb7fe700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 0x7f68cc0ffd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.126+0000 7f68cbfff700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f68c4007d70 tx=0x7f68c400a340 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.127+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c400ad40 con 0x7f68cc106df0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.127+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f68c40128e0 con 0x7f68cc106df0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.127+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68cc1a5a70 con 0x7f68cc106df0 2024-04-06T14:03:32.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.127+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68c40116b0 con 0x7f68cc106df0 2024-04-06T14:03:32.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.127+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68cc1a5f70 con 0x7f68cc106df0 2024-04-06T14:03:32.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.128+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f68cc1092f0 con 0x7f68cc106df0 2024-04-06T14:03:32.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.129+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f68c4012420 con 0x7f68cc106df0 2024-04-06T14:03:32.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.129+0000 7f68c8ff9700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 0x7f68b40a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:32.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.129+0000 7f68cb7fe700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 0x7f68b40a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:32.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.129+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f68c40cc660 con 0x7f68cc106df0 2024-04-06T14:03:32.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.130+0000 7f68cb7fe700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 0x7f68b40a35e0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f68bc009fd0 tx=0x7f68bc005c20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:32.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.134+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f68c4096290 con 0x7f68cc106df0 2024-04-06T14:03:32.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.325+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f68cc101250 con 0x7f68cc106df0 2024-04-06T14:03:32.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.325+0000 7f68c8ff9700 1 -- 172.21.15.90:0/1362458938 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v77) v1 ==== 93+0+10 (secure 0 0 0) 0x7f68c40998b0 con 0x7f68cc106df0 2024-04-06T14:03:32.326 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 msgr2=0x7f68b40a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 0x7f68b40a35e0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f68bc009fd0 tx=0x7f68bc005c20 comp rx=0 tx=0).stop 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 msgr2=0x7f68cc1007f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f68c4007d70 tx=0x7f68c400a340 comp rx=0 tx=0).stop 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 shutdown_connections 2024-04-06T14:03:32.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f68b40a1120 0x7f68b40a35e0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68cc106df0 0x7f68cc1007f0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68cc069020 0x7f68cc0ffd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 --2- 172.21.15.90:0/1362458938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68cc0699e0 0x7f68cc1002b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.327+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 >> 172.21.15.90:0/1362458938 conn(0x7f68cc076c80 msgr2=0x7f68cc0fe950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.328+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 shutdown_connections 2024-04-06T14:03:32.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:32.328+0000 7f68d1deb700 1 -- 172.21.15.90:0/1362458938 wait complete. 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.a on smithi090 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.d on smithi090 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.g on smithi090 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.j on smithi090 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.b on smithi105 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.e on smithi105 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.h on smithi105 2024-04-06T14:03:32.700 INFO:tasks.cephadm:Adding mds.k on smithi105 2024-04-06T14:03:32.701 INFO:tasks.cephadm:Adding mds.c on smithi134 2024-04-06T14:03:32.701 INFO:tasks.cephadm:Adding mds.f on smithi134 2024-04-06T14:03:32.701 INFO:tasks.cephadm:Adding mds.i on smithi134 2024-04-06T14:03:32.701 INFO:tasks.cephadm:Adding mds.l on smithi134 2024-04-06T14:03:32.701 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph orch apply mds all '12;smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l' 2024-04-06T14:03:32.938 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:03:33.314 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:33 smithi134 ceph-mon[47549]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:33.314 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:33 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1362458938' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:33 smithi105 ceph-mon[46969]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:33 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1362458938' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:33 smithi090 ceph-mon[43420]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:33 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1362458938' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:33.827 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 -- 172.21.15.134:0/136496218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 msgr2=0x7fbd4c0ff4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 --2- 172.21.15.134:0/136496218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c0ff4f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fbd3c009990 tx=0x7fbd3c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 -- 172.21.15.134:0/136496218 shutdown_connections 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 --2- 172.21.15.134:0/136496218 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c102a50 0x7fbd4c109300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 --2- 172.21.15.134:0/136496218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c0ff4f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 --2- 172.21.15.134:0/136496218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c1020b0 0x7fbd4c102510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 -- 172.21.15.134:0/136496218 >> 172.21.15.134:0/136496218 conn(0x7fbd4c0faa50 msgr2=0x7fbd4c0fce70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 -- 172.21.15.134:0/136496218 shutdown_connections 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 -- 172.21.15.134:0/136496218 wait complete. 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.826+0000 7fbd510a9700 1 Processor -- start 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 -- start start 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c1020b0 0x7fbd4c197330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:33.828 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c102a50 0x7fbd4c19e360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd4c19e930 con 0x7fbd4c1020b0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd4c19ea70 con 0x7fbd4c102a50 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd510a9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd4c19ec00 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.134:57970/0 (socket says 172.21.15.134:57970) 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 -- 172.21.15.134:0/2677185318 learned_addr learned my addr 172.21.15.134:0/2677185318 (peer_addr_for_me v2:172.21.15.134:0/0) 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4a59c700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c1020b0 0x7fbd4c197330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4b59e700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c102a50 0x7fbd4c19e360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 -- 172.21.15.134:0/2677185318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c102a50 msgr2=0x7fbd4c19e360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c102a50 0x7fbd4c19e360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 -- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c1020b0 msgr2=0x7fbd4c197330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c1020b0 0x7fbd4c197330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 -- 172.21.15.134:0/2677185318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd3c0095f0 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.829 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd4ad9d700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fbd3c00b5c0 tx=0x7fbd3c00f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:33.830 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.827+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd3c01d070 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.830 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.828+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd4c19ee80 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.830 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.828+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd3c022470 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.830 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.828+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd3c017650 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.830 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.828+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd4c19f1e0 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.829+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbd4c108380 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.830+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fbd3c0225e0 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.830+0000 7fbd33fff700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 0x7fbd340a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.831+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fbd3c0cdaf0 con 0x7fbd4c0ff0d0 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.831+0000 7fbd4a59c700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 0x7fbd340a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:33.831 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.831+0000 7fbd4a59c700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 0x7fbd340a35e0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fbd40009a10 tx=0x7fbd40009190 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:33.834 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:33.835+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbd3c0978d0 con 0x7fbd4c0ff0d0 2024-04-06T14:03:34.034 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.035+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l", "target": ["mon-mgr", ""]}) v1 -- 0x7fbd4c02ce20 con 0x7fbd340a1120 2024-04-06T14:03:34.042 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.043+0000 7fbd33fff700 1 -- 172.21.15.134:0/2677185318 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+28 (secure 0 0 0) 0x7fbd4c02ce20 con 0x7fbd340a1120 2024-04-06T14:03:34.042 INFO:teuthology.orchestra.run.smithi134.stdout:Scheduled mds.all update... 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 msgr2=0x7fbd340a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 0x7fbd340a35e0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fbd40009a10 tx=0x7fbd40009190 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 msgr2=0x7fbd4c196df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fbd3c00b5c0 tx=0x7fbd3c00f690 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 shutdown_connections 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbd340a1120 0x7fbd340a35e0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd4c102a50 0x7fbd4c19e360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd4c0ff0d0 0x7fbd4c196df0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 --2- 172.21.15.134:0/2677185318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd4c1020b0 0x7fbd4c197330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 >> 172.21.15.134:0/2677185318 conn(0x7fbd4c0faa50 msgr2=0x7fbd4c1054c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 shutdown_connections 2024-04-06T14:03:34.045 INFO:teuthology.orchestra.run.smithi134.stderr:2024-04-06T14:03:34.045+0000 7fbd510a9700 1 -- 172.21.15.134:0/2677185318 wait complete. 2024-04-06T14:03:34.418 DEBUG:teuthology.orchestra.run.smithi090:mds.a> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.a.service 2024-04-06T14:03:34.421 DEBUG:teuthology.orchestra.run.smithi090:mds.d> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.d.service 2024-04-06T14:03:34.452 INFO:journalctl@ceph.mds.a.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:03:34.453 DEBUG:teuthology.orchestra.run.smithi090:mds.g> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.g.service 2024-04-06T14:03:34.480 INFO:journalctl@ceph.mds.d.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:03:34.481 DEBUG:teuthology.orchestra.run.smithi090:mds.j> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.j.service 2024-04-06T14:03:34.507 INFO:journalctl@ceph.mds.g.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:03:34.508 DEBUG:teuthology.orchestra.run.smithi105:mds.b> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.b.service 2024-04-06T14:03:34.511 DEBUG:teuthology.orchestra.run.smithi105:mds.e> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.e.service 2024-04-06T14:03:34.534 INFO:journalctl@ceph.mds.j.smithi090.stdout:-- Logs begin at Sat 2024-04-06 13:42:34 UTC. -- 2024-04-06T14:03:34.550 INFO:journalctl@ceph.mds.b.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.551 DEBUG:teuthology.orchestra.run.smithi105:mds.h> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.h.service 2024-04-06T14:03:34.581 INFO:journalctl@ceph.mds.e.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.582 DEBUG:teuthology.orchestra.run.smithi105:mds.k> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.k.service 2024-04-06T14:03:34.609 INFO:journalctl@ceph.mds.h.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.610 DEBUG:teuthology.orchestra.run.smithi134:mds.c> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.c.service 2024-04-06T14:03:34.612 DEBUG:teuthology.orchestra.run.smithi134:mds.f> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.f.service 2024-04-06T14:03:34.638 INFO:journalctl@ceph.mds.k.smithi105.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.645 INFO:journalctl@ceph.mds.c.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.646 DEBUG:teuthology.orchestra.run.smithi134:mds.i> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.i.service 2024-04-06T14:03:34.675 INFO:journalctl@ceph.mds.f.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.676 DEBUG:teuthology.orchestra.run.smithi134:mds.l> sudo journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.l.service 2024-04-06T14:03:34.703 INFO:journalctl@ceph.mds.i.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.704 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-04-06T14:03:34.704 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:03:34.732 INFO:journalctl@ceph.mds.l.smithi134.stdout:-- Logs begin at Sat 2024-04-06 13:41:49 UTC. -- 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 -- 172.21.15.90:0/346378467 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 msgr2=0x7f66a40fff90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 --2- 172.21.15.90:0/346378467 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a40fff90 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f66940099c0 tx=0x7f6694009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 -- 172.21.15.90:0/346378467 shutdown_connections 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 --2- 172.21.15.90:0/346378467 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4101700 0x7f66a4107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 --2- 172.21.15.90:0/346378467 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a40fff90 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 --2- 172.21.15.90:0/346378467 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4100d60 0x7f66a41011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 -- 172.21.15.90:0/346378467 >> 172.21.15.90:0/346378467 conn(0x7f66a40fb100 msgr2=0x7f66a40fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 -- 172.21.15.90:0/346378467 shutdown_connections 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.844+0000 7f66aaa26700 1 -- 172.21.15.90:0/346378467 wait complete. 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 Processor -- start 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 -- start start 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:34.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4100d60 0x7f66a4102140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4101700 0x7f66a4102680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a4102e20 con 0x7f66a40ffb70 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a4102f90 con 0x7f66a4100d60 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66aaa26700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a41a0c90 con 0x7f66a4101700 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37820/0 (socket says 172.21.15.90:37820) 2024-04-06T14:03:34.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66a3fff700 1 -- 172.21.15.90:0/3538507686 learned_addr learned my addr 172.21.15.90:0/3538507686 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66a37fe700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4100d60 0x7f66a4102140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.845+0000 7f66a8fc3700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4101700 0x7f66a4102680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 -- 172.21.15.90:0/3538507686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4101700 msgr2=0x7f66a4102680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4101700 0x7f66a4102680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 -- 172.21.15.90:0/3538507686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4100d60 msgr2=0x7f66a4102140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4100d60 0x7f66a4102140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 -- 172.21.15.90:0/3538507686 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66940095f0 con 0x7f66a40ffb70 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a3fff700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f6694009fd0 tx=0x7f6694005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669401d070 con 0x7f66a40ffb70 2024-04-06T14:03:34.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66a41a0ec0 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f66940059a0 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f66940173f0 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.846+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66a41a1280 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.847+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f66a4067a90 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.848+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f6694022600 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.849+0000 7f66a17fa700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 0x7f668c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.849+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f66940cdb70 con 0x7f66a40ffb70 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.849+0000 7f66a37fe700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 0x7f668c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:34.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.849+0000 7f66a37fe700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 0x7f668c0a35e0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6698009c60 tx=0x7f6698009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:34.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:34.852+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6694097830 con 0x7f66a40ffb70 2024-04-06T14:03:35.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.051+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f66a410b380 con 0x7f66a40ffb70 2024-04-06T14:03:35.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.052+0000 7f66a17fa700 1 -- 172.21.15.90:0/3538507686 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v77) v1 ==== 74+0+18411 (secure 0 0 0) 0x7f669409ae50 con 0x7f66a40ffb70 2024-04-06T14:03:35.055 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:35.055 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":77,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:03:26.970706+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 msgr2=0x7f668c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 0x7f668c0a35e0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f6698009c60 tx=0x7f6698009260 comp rx=0 tx=0).stop 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 msgr2=0x7f66a4103ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f6694009fd0 tx=0x7f6694005f10 comp rx=0 tx=0).stop 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 shutdown_connections 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f668c0a1120 0x7f668c0a35e0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4101700 0x7f66a4102680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a40ffb70 0x7f66a4103ab0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 --2- 172.21.15.90:0/3538507686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4100d60 0x7f66a4102140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 >> 172.21.15.90:0/3538507686 conn(0x7f66a40fb100 msgr2=0x7f66a4067e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:35.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 shutdown_connections 2024-04-06T14:03:35.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.054+0000 7f66aaa26700 1 -- 172.21.15.90:0/3538507686 wait complete. 2024-04-06T14:03:35.068 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:03:35.068 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 -- 172.21.15.90:0/1918246922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 msgr2=0x7f50f81091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 --2- 172.21.15.90:0/1918246922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f81091d0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f50f000b0d0 tx=0x7f50f000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 -- 172.21.15.90:0/1918246922 shutdown_connections 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 --2- 172.21.15.90:0/1918246922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f81091d0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 --2- 172.21.15.90:0/1918246922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f81011b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 --2- 172.21.15.90:0/1918246922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f81023e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 -- 172.21.15.90:0/1918246922 >> 172.21.15.90:0/1918246922 conn(0x7f50f80fc300 msgr2=0x7f50f80fe740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 -- 172.21.15.90:0/1918246922 shutdown_connections 2024-04-06T14:03:35.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.199+0000 7f50fde6b700 1 -- 172.21.15.90:0/1918246922 wait complete. 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 Processor -- start 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 -- start start 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f8196e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f8197340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50f819e930 con 0x7f50f8102920 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50f819ea70 con 0x7f50f8101f80 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50fde6b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50f819ec00 con 0x7f50f8100d90 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37832/0 (socket says 172.21.15.90:37832) 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 -- 172.21.15.90:0/506338483 learned_addr learned my addr 172.21.15.90:0/506338483 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f77fe700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f8196e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f6ffd700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f8197340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 -- 172.21.15.90:0/506338483 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 msgr2=0x7f50f8196e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f8196e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 -- 172.21.15.90:0/506338483 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 msgr2=0x7f50f8197340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f8197340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 -- 172.21.15.90:0/506338483 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50f0009d00 con 0x7f50f8102920 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f77fe700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f8196e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f6ffd700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f8197340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.200+0000 7f50f7fff700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f50f0007b60 tx=0x7f50f0008f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f50f000e040 con 0x7f50f8102920 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f50f000be70 con 0x7f50f8102920 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f50f819ee80 con 0x7f50f8102920 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f50f001bb00 con 0x7f50f8102920 2024-04-06T14:03:35.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f50f819f2a0 con 0x7f50f8102920 2024-04-06T14:03:35.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.201+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f50f8108140 con 0x7f50f8102920 2024-04-06T14:03:35.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.203+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f50f001d670 con 0x7f50f8102920 2024-04-06T14:03:35.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.203+0000 7f50f4ff9700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 0x7f50e40a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.203+0000 7f50f77fe700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 0x7f50e40a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.203+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f50f00ce190 con 0x7f50f8102920 2024-04-06T14:03:35.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.204+0000 7f50f77fe700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 0x7f50e40a35e0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f50f8101de0 tx=0x7f50e0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:35.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.207+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f50f0097e50 con 0x7f50f8102920 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='client.24418 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: Saving service mds.all spec with placement smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l;count:12 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.364 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:35.365 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:35.365 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:35 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.401+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f50f810c830 con 0x7f50f8102920 2024-04-06T14:03:35.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.402+0000 7f50f4ff9700 1 -- 172.21.15.90:0/506338483 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v77) v1 ==== 93+0+10 (secure 0 0 0) 0x7f50f009b470 con 0x7f50f8102920 2024-04-06T14:03:35.413 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:03:35.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 msgr2=0x7f50e40a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 0x7f50e40a35e0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f50f8101de0 tx=0x7f50e0009260 comp rx=0 tx=0).stop 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 msgr2=0x7f50f819e360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f50f0007b60 tx=0x7f50f0008f40 comp rx=0 tx=0).stop 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 shutdown_connections 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f50e40a1120 0x7f50e40a35e0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f50f8102920 0x7f50f819e360 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f50f8100d90 0x7f50f8196e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 --2- 172.21.15.90:0/506338483 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f50f8101f80 0x7f50f8197340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 >> 172.21.15.90:0/506338483 conn(0x7f50f80fc300 msgr2=0x7f50f8105280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:35.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 shutdown_connections 2024-04-06T14:03:35.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.404+0000 7f50fde6b700 1 -- 172.21.15.90:0/506338483 wait complete. 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='client.24418 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: Saving service mds.all spec with placement smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l;count:12 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:35.417 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:35 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.420 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='client.24418 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: Saving service mds.all spec with placement smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l;count:12 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:35 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:35.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- 172.21.15.90:0/2824021033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c100c90 msgr2=0x7f564c1010b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 --2- 172.21.15.90:0/2824021033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c100c90 0x7f564c1010b0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f56480099c0 tx=0x7f5648009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- 172.21.15.90:0/2824021033 shutdown_connections 2024-04-06T14:03:35.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 --2- 172.21.15.90:0/2824021033 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c102820 0x7f564c1090d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 --2- 172.21.15.90:0/2824021033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c100c90 0x7f564c1010b0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 --2- 172.21.15.90:0/2824021033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c101e80 0x7f564c1022e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- 172.21.15.90:0/2824021033 >> 172.21.15.90:0/2824021033 conn(0x7f564c0fc220 msgr2=0x7f564c0fe640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- 172.21.15.90:0/2824021033 shutdown_connections 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- 172.21.15.90:0/2824021033 wait complete. 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 Processor -- start 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.542+0000 7f56543cb700 1 -- start start 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c100c90 0x7f564c074b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c102820 0x7f564c0736e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f564c073e80 con 0x7f564c101e80 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f564c073ff0 con 0x7f564c102820 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f564c1a0c30 con 0x7f564c100c90 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37852/0 (socket says 172.21.15.90:37852) 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 -- 172.21.15.90:0/2130053883 learned_addr learned my addr 172.21.15.90:0/2130053883 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5652167700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c100c90 0x7f564c074b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5652968700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c102820 0x7f564c0736e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 -- 172.21.15.90:0/2130053883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c100c90 msgr2=0x7f564c074b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c100c90 0x7f564c074b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 -- 172.21.15.90:0/2130053883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c102820 msgr2=0x7f564c0736e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c102820 0x7f564c0736e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 -- 172.21.15.90:0/2130053883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56480095f0 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f5651966700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f563c00c920 tx=0x7f563c00cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f563c007aa0 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f564c1a0ec0 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f563c00ce70 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f563c005440 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.543+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f564c1a1340 con 0x7f564c101e80 2024-04-06T14:03:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.544+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f564c108130 con 0x7f564c101e80 2024-04-06T14:03:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.546+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f563c0055a0 con 0x7f564c101e80 2024-04-06T14:03:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.547+0000 7f56437fe700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 0x7f56380a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.547+0000 7f5652167700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 0x7f56380a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.547+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f563c0cc3b0 con 0x7f564c101e80 2024-04-06T14:03:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.547+0000 7f5652167700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 0x7f56380a35e0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f5648003730 tx=0x7f5648000bc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.550+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f563c095f40 con 0x7f564c101e80 2024-04-06T14:03:35.643 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:03:35 smithi134 systemd[1]: Starting Ceph mds.l for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:35.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.790+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f564c067a20 con 0x7f564c101e80 2024-04-06T14:03:35.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.791+0000 7f56437fe700 1 -- 172.21.15.90:0/2130053883 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 1 v1) v1 ==== 93+0+501 (secure 0 0 0) 0x7f563c01d210 con 0x7f564c101e80 2024-04-06T14:03:35.792 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:35.792 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 msgr2=0x7f56380a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 0x7f56380a35e0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f5648003730 tx=0x7f5648000bc0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 msgr2=0x7f564c0731a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f563c00c920 tx=0x7f563c00cc30 comp rx=0 tx=0).stop 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 shutdown_connections 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56380a1120 0x7f56380a35e0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f564c102820 0x7f564c0736e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f564c100c90 0x7f564c074b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 --2- 172.21.15.90:0/2130053883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f564c101e80 0x7f564c0731a0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:35.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 >> 172.21.15.90:0/2130053883 conn(0x7f564c0fc220 msgr2=0x7f564c105270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:35.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 shutdown_connections 2024-04-06T14:03:35.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:35.793+0000 7f56543cb700 1 -- 172.21.15.90:0/2130053883 wait complete. 2024-04-06T14:03:35.795 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 1 2024-04-06T14:03:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: Deploying daemon mds.l on smithi134 2024-04-06T14:03:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3538507686' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/506338483' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2130053883' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6832/1837725694,v1:172.21.15.134:6833/1837725694] up:boot 2024-04-06T14:03:36.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: fsmap 1 up:standby 2024-04-06T14:03:36.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:36 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:03:35 smithi134 bash[84341]: fd3906325ed306de83b6d6809ef2370fc0736002c5c4adb14b1ff1f7f3d5339f 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:03:35 smithi134 systemd[1]: Started Ceph mds.l for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mds[84379]: -- 172.21.15.134:0/4113313939 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fc0a2401a0 con 0x55fc0a216800 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-l[84356]: starting mds.l at 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: Deploying daemon mds.l on smithi134 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3538507686' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/506338483' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2130053883' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6832/1837725694,v1:172.21.15.134:6833/1837725694] up:boot 2024-04-06T14:03:36.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: fsmap 1 up:standby 2024-04-06T14:03:36.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:36 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:03:36.458 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: Deploying daemon mds.l on smithi134 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3538507686' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/506338483' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2130053883' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6832/1837725694,v1:172.21.15.134:6833/1837725694] up:boot 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: fsmap 1 up:standby 2024-04-06T14:03:36.459 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:36 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: Deploying daemon mds.k on smithi105 2024-04-06T14:03:37.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mon[46969]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: Deploying daemon mds.k on smithi105 2024-04-06T14:03:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:37 smithi134 ceph-mon[47549]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: Deploying daemon mds.k on smithi105 2024-04-06T14:03:37.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:37 smithi090 ceph-mon[43420]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:37.710 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:03:37 smithi105 systemd[1]: Starting Ceph mds.k for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:37.807 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:37.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.947+0000 7f32e09a7700 1 -- 172.21.15.90:0/2841830522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 msgr2=0x7f32dc100d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.947+0000 7f32e09a7700 1 --2- 172.21.15.90:0/2841830522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc100d10 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f32d400b0d0 tx=0x7f32d400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 -- 172.21.15.90:0/2841830522 shutdown_connections 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 --2- 172.21.15.90:0/2841830522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc100d10 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 --2- 172.21.15.90:0/2841830522 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 0x7f32dc0ff990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 --2- 172.21.15.90:0/2841830522 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 0x7f32dc100350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 -- 172.21.15.90:0/2841830522 >> 172.21.15.90:0/2841830522 conn(0x7f32dc0fb110 msgr2=0x7f32dc0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 -- 172.21.15.90:0/2841830522 shutdown_connections 2024-04-06T14:03:37.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.948+0000 7f32e09a7700 1 -- 172.21.15.90:0/2841830522 wait complete. 2024-04-06T14:03:37.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 Processor -- start 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 -- start start 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 0x7f32dc198e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 0x7f32dc199360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32dc1a0950 con 0x7f32dc100890 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32dc1a0a90 con 0x7f32dc0ffed0 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32e09a7700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32dc1a0c20 con 0x7f32dc0ff570 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37870/0 (socket says 172.21.15.90:37870) 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 -- 172.21.15.90:0/4102235381 learned_addr learned my addr 172.21.15.90:0/4102235381 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32d9d9b700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 0x7f32dc199360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32da59c700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 0x7f32dc198e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:37.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 -- 172.21.15.90:0/4102235381 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 msgr2=0x7f32dc198e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 0x7f32dc198e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 -- 172.21.15.90:0/4102235381 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 msgr2=0x7f32dc199360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 0x7f32dc199360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.949+0000 7f32dad9d700 1 -- 172.21.15.90:0/4102235381 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32d4009d00 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32dad9d700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f32d4007bd0 tx=0x7f32d4009440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32d400e040 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f32d401dd50 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32dc1a0ea0 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32d40123f0 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.950+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32dc1a12c0 con 0x7f32dc100890 2024-04-06T14:03:37.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.951+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32dc10a490 con 0x7f32dc100890 2024-04-06T14:03:37.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.952+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f32d401d890 con 0x7f32dc100890 2024-04-06T14:03:37.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.953+0000 7f32cb7fe700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 0x7f32c40a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:37.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.953+0000 7f32da59c700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 0x7f32c40a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:37.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.953+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f32d40ce9d0 con 0x7f32dc100890 2024-04-06T14:03:37.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.953+0000 7f32da59c700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 0x7f32c40a35e0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f32cc00ac50 tx=0x7f32cc00a380 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:37.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:37.957+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f32d4098690 con 0x7f32dc100890 2024-04-06T14:03:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.212+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f32dc067190 con 0x7f32dc100890 2024-04-06T14:03:38.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.213+0000 7f32cb7fe700 1 -- 172.21.15.90:0/4102235381 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) v1 ==== 93+0+1260 (secure 0 0 0) 0x7f32d409bcb0 con 0x7f32dc100890 2024-04-06T14:03:38.214 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:38.214 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 msgr2=0x7f32c40a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 0x7f32c40a35e0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f32cc00ac50 tx=0x7f32cc00a380 comp rx=0 tx=0).stop 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 msgr2=0x7f32dc1a0380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f32d4007bd0 tx=0x7f32d4009440 comp rx=0 tx=0).stop 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 shutdown_connections 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f32c40a1120 0x7f32c40a35e0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32dc100890 0x7f32dc1a0380 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32dc0ff570 0x7f32dc198e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 --2- 172.21.15.90:0/4102235381 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32dc0ffed0 0x7f32dc199360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:38.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 >> 172.21.15.90:0/4102235381 conn(0x7f32dc0fb110 msgr2=0x7f32dc0fc760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:38.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 shutdown_connections 2024-04-06T14:03:38.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:38.215+0000 7f32e09a7700 1 -- 172.21.15.90:0/4102235381 wait complete. 2024-04-06T14:03:38.217 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 2 2024-04-06T14:03:38.246 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-mds[83408]: -- 172.21.15.105:0/2053841673 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557c19e141a0 con 0x557c19deb000 2024-04-06T14:03:38.247 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:03:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-k[83385]: starting mds.k at 2024-04-06T14:03:38.247 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:03:37 smithi105 bash[83369]: 841e9e3f2b3133a6d0886287890e409427903a9ad1c8ffb27f628f3db18e86bb 2024-04-06T14:03:38.247 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:03:37 smithi105 systemd[1]: Started Ceph mds.k for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: Deploying daemon mds.j on smithi090 2024-04-06T14:03:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4102235381' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6832/2124581639,v1:172.21.15.105:6833/2124581639] up:boot 2024-04-06T14:03:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: fsmap 2 up:standby 2024-04-06T14:03:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:38 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: Deploying daemon mds.j on smithi090 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4102235381' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6832/2124581639,v1:172.21.15.105:6833/2124581639] up:boot 2024-04-06T14:03:39.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: fsmap 2 up:standby 2024-04-06T14:03:39.225 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:38 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: Deploying daemon mds.j on smithi090 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4102235381' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6832/2124581639,v1:172.21.15.105:6833/2124581639] up:boot 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: fsmap 2 up:standby 2024-04-06T14:03:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:38 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:03:39.538 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:03:39 smithi090 systemd[1]: Starting Ceph mds.j for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:40.038 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:03:39 smithi090 bash[93882]: 6e9b4f7e3781e0a7664b3d2548625a7ddab504faf8b8d203cb98b2678289a3d1 2024-04-06T14:03:40.038 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:03:39 smithi090 systemd[1]: Started Ceph mds.j for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:40.038 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:03:39 smithi090 ceph-mds[93920]: -- 172.21.15.90:0/1831785545 <== mon.0 v2:172.21.15.90:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5639606341a0 con 0x56396060a800 2024-04-06T14:03:40.038 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:03:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-j[93897]: starting mds.j at 2024-04-06T14:03:40.229 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:40.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 -- 172.21.15.90:0/3929589243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5840ffb70 msgr2=0x7fc5840fff90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 --2- 172.21.15.90:0/3929589243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5840ffb70 0x7fc5840fff90 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fc5740099c0 tx=0x7fc574009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 -- 172.21.15.90:0/3929589243 shutdown_connections 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 --2- 172.21.15.90:0/3929589243 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584101700 0x7fc584107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 --2- 172.21.15.90:0/3929589243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5840ffb70 0x7fc5840fff90 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 --2- 172.21.15.90:0/3929589243 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc584100d60 0x7fc5841011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 -- 172.21.15.90:0/3929589243 >> 172.21.15.90:0/3929589243 conn(0x7fc5840fb100 msgr2=0x7fc5840fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 -- 172.21.15.90:0/3929589243 shutdown_connections 2024-04-06T14:03:40.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.384+0000 7fc588989700 1 -- 172.21.15.90:0/3929589243 wait complete. 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 Processor -- start 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 -- start start 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5840ffb70 0x7fc584068980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 0x7fc58419b860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc58419e650 con 0x7fc584101700 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc58419e7c0 con 0x7fc5840ffb70 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc588989700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc58419e980 con 0x7fc584100d60 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37898/0 (socket says 172.21.15.90:37898) 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 -- 172.21.15.90:0/3783682499 learned_addr learned my addr 172.21.15.90:0/3783682499 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:40.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc58259c700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5840ffb70 0x7fc584068980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc581d9b700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 0x7fc58419b860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 -- 172.21.15.90:0/3783682499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 msgr2=0x7fc58419b860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 0x7fc58419b860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 -- 172.21.15.90:0/3783682499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5840ffb70 msgr2=0x7fc584068980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5840ffb70 0x7fc584068980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc582d9d700 1 -- 172.21.15.90:0/3783682499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5740095f0 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.385+0000 7fc581d9b700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 0x7fc58419b860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc582d9d700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fc57c00ada0 tx=0x7fc57c00c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc57c00f070 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc58419ec60 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc57c007b30 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc57c0053e0 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.386+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc58419f170 con 0x7fc584101700 2024-04-06T14:03:40.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.387+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc584106f40 con 0x7fc584101700 2024-04-06T14:03:40.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.388+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fc57c00c980 con 0x7fc584101700 2024-04-06T14:03:40.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.388+0000 7fc5737fe700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 0x7fc56c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:40.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.388+0000 7fc58259c700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 0x7fc56c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:40.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.389+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fc57c0cc0a0 con 0x7fc584101700 2024-04-06T14:03:40.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.389+0000 7fc58259c700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 0x7fc56c0a35e0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fc574003820 tx=0x7fc574005040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:40.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.392+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc57c095cc0 con 0x7fc584101700 2024-04-06T14:03:40.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.644+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc584067a20 con 0x7fc584101700 2024-04-06T14:03:40.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.644+0000 7fc5737fe700 1 -- 172.21.15.90:0/3783682499 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 4 v4) v1 ==== 93+0+2777 (secure 0 0 0) 0x7fc57c0992e0 con 0x7fc584101700 2024-04-06T14:03:40.645 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:40.645 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":4,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-04-06T14:03:40.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 msgr2=0x7fc56c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:40.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 0x7fc56c0a35e0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fc574003820 tx=0x7fc574005040 comp rx=0 tx=0).stop 2024-04-06T14:03:40.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 msgr2=0x7fc58419deb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:40.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fc57c00ada0 tx=0x7fc57c00c330 comp rx=0 tx=0).stop 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 shutdown_connections 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc56c0a1120 0x7fc56c0a35e0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc584101700 0x7fc58419deb0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5840ffb70 0x7fc584068980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 --2- 172.21.15.90:0/3783682499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc584100d60 0x7fc58419b860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 >> 172.21.15.90:0/3783682499 conn(0x7fc5840fb100 msgr2=0x7fc584104080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 shutdown_connections 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:40.647+0000 7fc588989700 1 -- 172.21.15.90:0/3783682499 wait complete. 2024-04-06T14:03:40.648 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 4 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:03:41 smithi090 systemd[1]: Starting Ceph mds.d for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:41.031 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: Deploying daemon mds.d on smithi090 2024-04-06T14:03:41.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.90:6834/1262397636,v1:172.21.15.90:6835/1262397636] up:boot 2024-04-06T14:03:41.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: fsmap 3 up:standby 2024-04-06T14:03:41.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:03:41.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:41.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:40 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3783682499' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: Deploying daemon mds.d on smithi090 2024-04-06T14:03:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.90:6834/1262397636,v1:172.21.15.90:6835/1262397636] up:boot 2024-04-06T14:03:41.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: fsmap 3 up:standby 2024-04-06T14:03:41.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:03:41.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:41.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:40 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3783682499' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: Deploying daemon mds.d on smithi090 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.90:6834/1262397636,v1:172.21.15.90:6835/1262397636] up:boot 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: fsmap 3 up:standby 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:40 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3783682499' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:41.788 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:03:41 smithi090 ceph-mds[94307]: -- 172.21.15.90:0/3592116549 <== mon.0 v2:172.21.15.90:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d1fe28a1a0 con 0x55d1fe261000 2024-04-06T14:03:41.788 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:03:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-d[94284]: starting mds.d at 2024-04-06T14:03:41.788 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:03:41 smithi090 bash[94269]: 2898b7aa9e44564b8391a3b94d917924ba1a4512d7fa2e3b7dd6cc588a8c5137 2024-04-06T14:03:41.789 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:03:41 smithi090 systemd[1]: Started Ceph mds.d for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: Deploying daemon mds.e on smithi105 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.90:6836/886535758,v1:172.21.15.90:6837/886535758] up:boot 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: fsmap 4 up:standby 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:03:42.556 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:42 smithi105 ceph-mon[46969]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:42.660 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:42.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: Deploying daemon mds.e on smithi105 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.90:6836/886535758,v1:172.21.15.90:6837/886535758] up:boot 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: fsmap 4 up:standby 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:03:42.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:42 smithi090 ceph-mon[43420]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 -- 172.21.15.90:0/2233471346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988101e90 msgr2=0x7ff9881022f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 --2- 172.21.15.90:0/2233471346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988101e90 0x7ff9881022f0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7ff9840099c0 tx=0x7ff984009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 -- 172.21.15.90:0/2233471346 shutdown_connections 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 --2- 172.21.15.90:0/2233471346 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff9881090e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 --2- 172.21.15.90:0/2233471346 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988100ca0 0x7ff9881010c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 --2- 172.21.15.90:0/2233471346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988101e90 0x7ff9881022f0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 -- 172.21.15.90:0/2233471346 >> 172.21.15.90:0/2233471346 conn(0x7ff9880fc210 msgr2=0x7ff9880fe650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 -- 172.21.15.90:0/2233471346 shutdown_connections 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.800+0000 7ff9906aa700 1 -- 172.21.15.90:0/2233471346 wait complete. 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 Processor -- start 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 -- start start 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 0x7ff988197200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff98819e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff98819e7e0 con 0x7ff988100ca0 2024-04-06T14:03:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff98819e950 con 0x7ff988102830 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff9906aa700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff98819eb10 con 0x7ff988101e90 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff98e446700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff98e446700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:37936/0 (socket says 172.21.15.90:37936) 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff98e446700 1 -- 172.21.15.90:0/3165546822 learned_addr learned my addr 172.21.15.90:0/3165546822 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.801+0000 7ff98ec47700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff98819e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98dc45700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 0x7ff988197200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 -- 172.21.15.90:0/3165546822 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 msgr2=0x7ff988197200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 0x7ff988197200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 -- 172.21.15.90:0/3165546822 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 msgr2=0x7ff98819e210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff98819e210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 -- 172.21.15.90:0/3165546822 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9840095f0 con 0x7ff988100ca0 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98ec47700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff98819e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98dc45700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 0x7ff988197200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff98e446700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7ff97c00b500 tx=0x7ff97c00b8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff97c011830 con 0x7ff988100ca0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff97c011e50 con 0x7ff988100ca0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff97c00f2e0 con 0x7ff988100ca0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff98819edf0 con 0x7ff988100ca0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.802+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff98819f2a0 con 0x7ff988100ca0 2024-04-06T14:03:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.803+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff988108070 con 0x7ff988100ca0 2024-04-06T14:03:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.804+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7ff97c010580 con 0x7ff988100ca0 2024-04-06T14:03:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.804+0000 7ff97b7fe700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 0x7ff9740a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.805+0000 7ff98dc45700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 0x7ff9740a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.805+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7ff97c0cc470 con 0x7ff988100ca0 2024-04-06T14:03:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.805+0000 7ff98dc45700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 0x7ff9740a3660 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7ff984006010 tx=0x7ff984009f10 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:42.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:42.808+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff97c0960a0 con 0x7ff988100ca0 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: Deploying daemon mds.e on smithi105 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.90:6836/886535758,v1:172.21.15.90:6837/886535758] up:boot 2024-04-06T14:03:42.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: fsmap 4 up:standby 2024-04-06T14:03:42.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:03:42.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:42 smithi134 ceph-mon[47549]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:42.997 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:03:42 smithi105 systemd[1]: Starting Ceph mds.e for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:43.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.052+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff988067a20 con 0x7ff988100ca0 2024-04-06T14:03:43.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.052+0000 7ff97b7fe700 1 -- 172.21.15.90:0/3165546822 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) v1 ==== 93+0+3531 (secure 0 0 0) 0x7ff97c015de0 con 0x7ff988100ca0 2024-04-06T14:03:43.053 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:43.053 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 msgr2=0x7ff9740a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 0x7ff9740a3660 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7ff984006010 tx=0x7ff984009f10 comp rx=0 tx=0).stop 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 msgr2=0x7ff988196cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7ff97c00b500 tx=0x7ff97c00b8c0 comp rx=0 tx=0).stop 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 shutdown_connections 2024-04-06T14:03:43.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff9740a11a0 0x7ff9740a3660 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff988102830 0x7ff98819e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff988100ca0 0x7ff988196cc0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 --2- 172.21.15.90:0/3165546822 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff988101e90 0x7ff988197200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 >> 172.21.15.90:0/3165546822 conn(0x7ff9880fc210 msgr2=0x7ff9881051b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 shutdown_connections 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:43.054+0000 7ff9906aa700 1 -- 172.21.15.90:0/3165546822 wait complete. 2024-04-06T14:03:43.056 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 5 2024-04-06T14:03:43.497 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mds[83747]: -- 172.21.15.105:0/3643215782 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556ea2b441a0 con 0x556ea2b1b000 2024-04-06T14:03:43.497 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: starting mds.e at 2024-04-06T14:03:43.497 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:03:43 smithi105 bash[83709]: f1cfd968c501408466bacfeb8ccc41246bd4950dadc2a3fda7343f719adefd20 2024-04-06T14:03:43.497 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:03:43 smithi105 systemd[1]: Started Ceph mds.e for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:43.979 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3165546822' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:43.980 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:43 smithi134 ceph-mon[47549]: Deploying daemon mds.f on smithi134 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3165546822' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:44.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:44.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:43 smithi105 ceph-mon[46969]: Deploying daemon mds.f on smithi134 2024-04-06T14:03:44.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3165546822' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:44.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:44.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:44.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:44.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:44.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:43 smithi090 ceph-mon[43420]: Deploying daemon mds.f on smithi134 2024-04-06T14:03:44.643 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:03:44 smithi134 systemd[1]: Starting Ceph mds.f for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:45.068 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- 172.21.15.90:0/132602596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc100d60 msgr2=0x7f0edc1011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 --2- 172.21.15.90:0/132602596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc100d60 0x7f0edc1011c0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f0ed80099c0 tx=0x7f0ed8009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- 172.21.15.90:0/132602596 shutdown_connections 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 --2- 172.21.15.90:0/132602596 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc101700 0x7f0edc107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 --2- 172.21.15.90:0/132602596 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc0ffb70 0x7f0edc0fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 --2- 172.21.15.90:0/132602596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc100d60 0x7f0edc1011c0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- 172.21.15.90:0/132602596 >> 172.21.15.90:0/132602596 conn(0x7f0edc0fb100 msgr2=0x7f0edc0fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:45.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- 172.21.15.90:0/132602596 shutdown_connections 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- 172.21.15.90:0/132602596 wait complete. 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 Processor -- start 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.201+0000 7f0ee449e700 1 -- start start 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc100d60 0x7f0edc197120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 0x7f0edc19e0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0edc19e6c0 con 0x7f0edc0ffb70 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0edc19e800 con 0x7f0edc101700 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0edc19e990 con 0x7f0edc100d60 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51760/0 (socket says 172.21.15.90:51760) 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 -- 172.21.15.90:0/1542805096 learned_addr learned my addr 172.21.15.90:0/1542805096 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:45.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee1a39700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc100d60 0x7f0edc197120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee2a3b700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 0x7f0edc19e0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 -- 172.21.15.90:0/1542805096 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc100d60 msgr2=0x7f0edc197120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc100d60 0x7f0edc197120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 -- 172.21.15.90:0/1542805096 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 msgr2=0x7f0edc19e0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 0x7f0edc19e0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 -- 172.21.15.90:0/1542805096 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ed80095f0 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee2a3b700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 0x7f0edc19e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee223a700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0ecc00d980 tx=0x7f0ecc00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ecc004020 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0ecc009730 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0edc19ec70 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ecc003a80 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.202+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0edc19f180 con 0x7f0edc0ffb70 2024-04-06T14:03:45.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.203+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0edc067a20 con 0x7f0edc0ffb70 2024-04-06T14:03:45.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.204+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f0ecc0098a0 con 0x7f0edc0ffb70 2024-04-06T14:03:45.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.205+0000 7f0ed37fe700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 0x7f0ec80a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:45.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.205+0000 7f0ee1a39700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 0x7f0ec80a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:45.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.205+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f0ecc0cc140 con 0x7f0edc0ffb70 2024-04-06T14:03:45.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.205+0000 7f0ee1a39700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 0x7f0ec80a35e0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f0ed8009990 tx=0x7f0ed8000bc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:45.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.209+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0ecc095d70 con 0x7f0edc0ffb70 2024-04-06T14:03:45.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:44 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6834/3686963284,v1:172.21.15.105:6835/3686963284] up:boot 2024-04-06T14:03:45.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:44 smithi105 ceph-mon[46969]: fsmap 5 up:standby 2024-04-06T14:03:45.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:44 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:03:45.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:44 smithi105 ceph-mon[46969]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:44 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6834/3686963284,v1:172.21.15.105:6835/3686963284] up:boot 2024-04-06T14:03:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:44 smithi090 ceph-mon[43420]: fsmap 5 up:standby 2024-04-06T14:03:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:44 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:03:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:44 smithi090 ceph-mon[43420]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6834/3686963284,v1:172.21.15.105:6835/3686963284] up:boot 2024-04-06T14:03:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-mon[47549]: fsmap 5 up:standby 2024-04-06T14:03:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:03:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-mon[47549]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:45.393 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:03:44 smithi134 bash[84682]: 9dc0a17e0ad68fb791b93326be5963c24c6cb91063a57c948f9043733c9903dc 2024-04-06T14:03:45.394 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:03:44 smithi134 systemd[1]: Started Ceph mds.f for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:45.394 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-mds[84719]: -- 172.21.15.134:0/2984137308 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55813a8c21a0 con 0x55813a898c00 2024-04-06T14:03:45.394 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:03:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: starting mds.f at 2024-04-06T14:03:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.453+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0edc19f310 con 0x7f0edc0ffb70 2024-04-06T14:03:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.454+0000 7f0ed37fe700 1 -- 172.21.15.90:0/1542805096 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 6 v6) v1 ==== 93+0+4291 (secure 0 0 0) 0x7f0ecc099390 con 0x7f0edc0ffb70 2024-04-06T14:03:45.455 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:45.455 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":6,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6}],"filesystems":[]} 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 msgr2=0x7f0ec80a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 0x7f0ec80a35e0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f0ed8009990 tx=0x7f0ed8000bc0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 msgr2=0x7f0edc196be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0ecc00d980 tx=0x7f0ecc00dc90 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 shutdown_connections 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0ec80a1120 0x7f0ec80a35e0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0edc101700 0x7f0edc19e0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0edc0ffb70 0x7f0edc196be0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 --2- 172.21.15.90:0/1542805096 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0edc100d60 0x7f0edc197120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 >> 172.21.15.90:0/1542805096 conn(0x7f0edc0fb100 msgr2=0x7f0edc067e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 shutdown_connections 2024-04-06T14:03:45.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:45.456+0000 7f0ee449e700 1 -- 172.21.15.90:0/1542805096 wait complete. 2024-04-06T14:03:45.458 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 6 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: Deploying daemon mds.c on smithi134 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1542805096' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6834/2168599536,v1:172.21.15.134:6835/2168599536] up:boot 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: fsmap 6 up:standby 2024-04-06T14:03:46.004 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:45 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: Deploying daemon mds.c on smithi134 2024-04-06T14:03:46.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1542805096' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:46.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6834/2168599536,v1:172.21.15.134:6835/2168599536] up:boot 2024-04-06T14:03:46.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: fsmap 6 up:standby 2024-04-06T14:03:46.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:45 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:03:46.254 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:03:46 smithi134 systemd[1]: Starting Ceph mds.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: Deploying daemon mds.c on smithi134 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1542805096' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6834/2168599536,v1:172.21.15.134:6835/2168599536] up:boot 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: fsmap 6 up:standby 2024-04-06T14:03:46.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:45 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:03:46.958 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:03:46 smithi134 bash[85032]: 45987d90b838df294e1432cbb89530aeecbddcd662e280261d00410c710a2a19 2024-04-06T14:03:46.958 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:03:46 smithi134 systemd[1]: Started Ceph mds.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:46.958 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mds[85069]: -- 172.21.15.134:0/1162545329 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563c8f84a1a0 con 0x563c8f820c00 2024-04-06T14:03:46.959 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: starting mds.c at 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:46 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:47.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:47.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:46 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:46 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:47.469 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 -- 172.21.15.90:0/1324566407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab680ffb70 msgr2=0x7fab680fff90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/1324566407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab680ffb70 0x7fab680fff90 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fab5c0099c0 tx=0x7fab5c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 -- 172.21.15.90:0/1324566407 shutdown_connections 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/1324566407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68101700 0x7fab68107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/1324566407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab680ffb70 0x7fab680fff90 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/1324566407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab68100d60 0x7fab681011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 -- 172.21.15.90:0/1324566407 >> 172.21.15.90:0/1324566407 conn(0x7fab680fb100 msgr2=0x7fab680fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:47.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 -- 172.21.15.90:0/1324566407 shutdown_connections 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.605+0000 7fab6f1bd700 1 -- 172.21.15.90:0/1324566407 wait complete. 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 Processor -- start 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 -- start start 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 0x7fab68196bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 0x7fab68197130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab6819e720 con 0x7fab68101700 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab6819e860 con 0x7fab68100d60 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6f1bd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab6819e9f0 con 0x7fab680ffb70 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab67fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 0x7fab68197130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51766/0 (socket says 172.21.15.90:51766) 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 -- 172.21.15.90:0/2077930182 learned_addr learned my addr 172.21.15.90:0/2077930182 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:47.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6cf59700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 0x7fab68196bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 -- 172.21.15.90:0/2077930182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 msgr2=0x7fab68196bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 0x7fab68196bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 -- 172.21.15.90:0/2077930182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 msgr2=0x7fab68197130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 0x7fab68197130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 -- 172.21.15.90:0/2077930182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab5c0095f0 con 0x7fab68101700 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6cf59700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 0x7fab68196bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab67fff700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 0x7fab68197130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab6d75a700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fab60007ee0 tx=0x7fab6000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.606+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab60017070 con 0x7fab68101700 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.607+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fab6000f040 con 0x7fab68101700 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.607+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab60005740 con 0x7fab68101700 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.607+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab6819ecd0 con 0x7fab68101700 2024-04-06T14:03:47.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.607+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab6819f190 con 0x7fab68101700 2024-04-06T14:03:47.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.608+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab68106fa0 con 0x7fab68101700 2024-04-06T14:03:47.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.609+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fab600058a0 con 0x7fab68101700 2024-04-06T14:03:47.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.609+0000 7fab65ffb700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 0x7fab580a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:47.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.610+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fab600cbd00 con 0x7fab68101700 2024-04-06T14:03:47.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.610+0000 7fab6cf59700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 0x7fab580a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:47.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.610+0000 7fab6cf59700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 0x7fab580a3660 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fab5c00b580 tx=0x7fab5c005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:47.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.614+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fab60095890 con 0x7fab68101700 2024-04-06T14:03:47.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.857+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fab68067a20 con 0x7fab68101700 2024-04-06T14:03:47.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.857+0000 7fab65ffb700 1 -- 172.21.15.90:0/2077930182 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+5808 (secure 0 0 0) 0x7fab60098eb0 con 0x7fab68101700 2024-04-06T14:03:47.858 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:47.858 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":8,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24457,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24469,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8}],"filesystems":[]} 2024-04-06T14:03:47.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 msgr2=0x7fab580a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:47.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 0x7fab580a3660 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fab5c00b580 tx=0x7fab5c005fb0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 msgr2=0x7fab6819e150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fab60007ee0 tx=0x7fab6000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 shutdown_connections 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fab580a11a0 0x7fab580a3660 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fab68101700 0x7fab6819e150 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fab680ffb70 0x7fab68196bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 --2- 172.21.15.90:0/2077930182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fab68100d60 0x7fab68197130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 >> 172.21.15.90:0/2077930182 conn(0x7fab680fb100 msgr2=0x7fab681040e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 shutdown_connections 2024-04-06T14:03:47.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:47.860+0000 7fab6f1bd700 1 -- 172.21.15.90:0/2077930182 wait complete. 2024-04-06T14:03:47.862 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 8 2024-04-06T14:03:48.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:47 smithi105 ceph-mon[46969]: Deploying daemon mds.h on smithi105 2024-04-06T14:03:48.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:47 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6836/659065731,v1:172.21.15.134:6837/659065731] up:boot 2024-04-06T14:03:48.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:47 smithi105 ceph-mon[46969]: fsmap 7 up:standby 2024-04-06T14:03:48.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:47 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:03:48.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:47 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2077930182' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:48.248 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:03:48 smithi105 systemd[1]: Starting Ceph mds.h for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:48.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:47 smithi090 ceph-mon[43420]: Deploying daemon mds.h on smithi105 2024-04-06T14:03:48.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:47 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6836/659065731,v1:172.21.15.134:6837/659065731] up:boot 2024-04-06T14:03:48.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:47 smithi090 ceph-mon[43420]: fsmap 7 up:standby 2024-04-06T14:03:48.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:47 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:03:48.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:47 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2077930182' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:48.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:47 smithi134 ceph-mon[47549]: Deploying daemon mds.h on smithi105 2024-04-06T14:03:48.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:47 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6836/659065731,v1:172.21.15.134:6837/659065731] up:boot 2024-04-06T14:03:48.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:47 smithi134 ceph-mon[47549]: fsmap 7 up:standby 2024-04-06T14:03:48.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:47 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:03:48.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:47 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2077930182' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:48.747 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mds[84097]: -- 172.21.15.105:0/1317395564 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56431d7221a0 con 0x56431d6f8800 2024-04-06T14:03:48.747 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: starting mds.h at 2024-04-06T14:03:48.747 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:03:48 smithi105 bash[84051]: 08ceb974d0edb2c75d388239f21cc2ce451b532fd905abf2dd4f3749cc913b4e 2024-04-06T14:03:48.747 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:03:48 smithi105 systemd[1]: Started Ceph mds.h for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:49.245 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:49.245 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.246 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.246 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.246 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:49.246 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:49.246 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:48 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:48 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:49.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:48 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:49.873 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:49.893 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:03:49 smithi134 systemd[1]: Starting Ceph mds.i for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 -- 172.21.15.90:0/2740140938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 msgr2=0x7f8c001064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 --2- 172.21.15.90:0/2740140938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c001064b0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f8bf4009990 tx=0x7f8bf4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 -- 172.21.15.90:0/2740140938 shutdown_connections 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 --2- 172.21.15.90:0/2740140938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 0x7f8c00108e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 --2- 172.21.15.90:0/2740140938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 0x7f8c00103b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 --2- 172.21.15.90:0/2740140938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c001064b0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 -- 172.21.15.90:0/2740140938 >> 172.21.15.90:0/2740140938 conn(0x7f8c000fb140 msgr2=0x7f8c000fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 -- 172.21.15.90:0/2740140938 shutdown_connections 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.008+0000 7f8c04da0700 1 -- 172.21.15.90:0/2740140938 wait complete. 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 Processor -- start 2024-04-06T14:03:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 -- start start 2024-04-06T14:03:50.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 0x7f8c00196cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 0x7f8c0019e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c0019e7c0 con 0x7f8c001040c0 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c0019e930 con 0x7f8c00101790 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8c04da0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c0019eaf0 con 0x7f8c001069f0 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8bfdd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8bfdd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51772/0 (socket says 172.21.15.90:51772) 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8bfdd9b700 1 -- 172.21.15.90:0/3782228399 learned_addr learned my addr 172.21.15.90:0/3782228399 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8bfed9d700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 0x7f8c0019e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.009+0000 7f8bfe59c700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 0x7f8c00196cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:50.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 -- 172.21.15.90:0/3782228399 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 msgr2=0x7f8c0019e1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 0x7f8c0019e1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 -- 172.21.15.90:0/3782228399 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 msgr2=0x7f8c00196cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 0x7f8c00196cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 -- 172.21.15.90:0/3782228399 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8bf40095f0 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bfdd9b700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f8bf40095c0 tx=0x7f8bf400f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8bf401d070 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c0019ed70 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8bf4005030 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8bf4004a50 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.010+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c0019f190 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.011+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c000fccc0 con 0x7f8c001040c0 2024-04-06T14:03:50.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.012+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f8bf4005450 con 0x7f8c001040c0 2024-04-06T14:03:50.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.013+0000 7f8bef7fe700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 0x7f8be80a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:50.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.013+0000 7f8bfe59c700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 0x7f8be80a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:50.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.013+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f8bf40cd6f0 con 0x7f8c001040c0 2024-04-06T14:03:50.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.013+0000 7f8bfe59c700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 0x7f8be80a35e0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f8c00198150 tx=0x7f8bf0009190 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:50.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.016+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8bf4097310 con 0x7f8c001040c0 2024-04-06T14:03:50.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:49 smithi105 ceph-mon[46969]: Deploying daemon mds.i on smithi134 2024-04-06T14:03:50.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:49 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:boot 2024-04-06T14:03:50.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:49 smithi105 ceph-mon[46969]: fsmap 8 up:standby 2024-04-06T14:03:50.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:49 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:03:50.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.266+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8c00067e70 con 0x7f8c001040c0 2024-04-06T14:03:50.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.266+0000 7f8bef7fe700 1 -- 172.21.15.90:0/3782228399 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) v1 ==== 93+0+6568 (secure 0 0 0) 0x7f8bf409a930 con 0x7f8c001040c0 2024-04-06T14:03:50.267 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:50.267 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24457,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24469,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24485,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9}],"filesystems":[]} 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 msgr2=0x7f8be80a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 0x7f8be80a35e0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f8c00198150 tx=0x7f8bf0009190 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 msgr2=0x7f8c00197210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f8bf40095c0 tx=0x7f8bf400f690 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 shutdown_connections 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8be80a1120 0x7f8be80a35e0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c001069f0 0x7f8c0019e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c00101790 0x7f8c00196cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 --2- 172.21.15.90:0/3782228399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c001040c0 0x7f8c00197210 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:50.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 >> 172.21.15.90:0/3782228399 conn(0x7f8c000fb140 msgr2=0x7f8c001020f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:50.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 shutdown_connections 2024-04-06T14:03:50.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:50.269+0000 7f8c04da0700 1 -- 172.21.15.90:0/3782228399 wait complete. 2024-04-06T14:03:50.271 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 9 2024-04-06T14:03:50.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:49 smithi090 ceph-mon[43420]: Deploying daemon mds.i on smithi134 2024-04-06T14:03:50.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:49 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:boot 2024-04-06T14:03:50.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:49 smithi090 ceph-mon[43420]: fsmap 8 up:standby 2024-04-06T14:03:50.282 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:49 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:03:50.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:49 smithi134 ceph-mon[47549]: Deploying daemon mds.i on smithi134 2024-04-06T14:03:50.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:49 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:boot 2024-04-06T14:03:50.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:49 smithi134 ceph-mon[47549]: fsmap 8 up:standby 2024-04-06T14:03:50.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:49 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:03:50.393 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:03:50 smithi134 ceph-mds[85412]: -- 172.21.15.134:0/1832080541 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fe1e0041a0 con 0x55fe1dfdb000 2024-04-06T14:03:50.394 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:03:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: starting mds.i at 2024-04-06T14:03:50.394 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:03:50 smithi134 bash[85374]: 04b8b17f22daf02a7ae89fe1bd0a21f9e1605b400b27336e4b54ee5e120979de 2024-04-06T14:03:50.394 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:03:50 smithi134 systemd[1]: Started Ceph mds.i for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:51.247 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:51.247 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.247 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.247 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: Deploying daemon mds.g on smithi090 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3782228399' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:boot 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: fsmap 9 up:standby 2024-04-06T14:03:51.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: Deploying daemon mds.g on smithi090 2024-04-06T14:03:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3782228399' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:boot 2024-04-06T14:03:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: fsmap 9 up:standby 2024-04-06T14:03:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:51 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:03:51.538 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:03:51 smithi090 systemd[1]: Starting Ceph mds.g for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: Deploying daemon mds.g on smithi090 2024-04-06T14:03:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3782228399' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:51.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:boot 2024-04-06T14:03:51.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: fsmap 9 up:standby 2024-04-06T14:03:51.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:51 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:03:52.038 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-mds[94836]: -- 172.21.15.90:0/3222964403 <== mon.0 v2:172.21.15.90:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560bdbc301a0 con 0x560bdbc06800 2024-04-06T14:03:52.038 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:03:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-g[94814]: starting mds.g at 2024-04-06T14:03:52.038 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:03:51 smithi090 bash[94799]: fff61848e6895f8f29e0f61a9b4227523da2754dbf77e1b9462fdf6b081c5f2e 2024-04-06T14:03:52.038 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:03:51 smithi090 systemd[1]: Started Ceph mds.g for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:52.284 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:52.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 -- 172.21.15.90:0/618822310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24100d60 msgr2=0x7f9c241011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 --2- 172.21.15.90:0/618822310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24100d60 0x7f9c241011c0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f9c140099c0 tx=0x7f9c14009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 -- 172.21.15.90:0/618822310 shutdown_connections 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 --2- 172.21.15.90:0/618822310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24101700 0x7f9c24107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 --2- 172.21.15.90:0/618822310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 0x7f9c240fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.417+0000 7f9c2977e700 1 --2- 172.21.15.90:0/618822310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24100d60 0x7f9c241011c0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- 172.21.15.90:0/618822310 >> 172.21.15.90:0/618822310 conn(0x7f9c240fb100 msgr2=0x7f9c240fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- 172.21.15.90:0/618822310 shutdown_connections 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- 172.21.15.90:0/618822310 wait complete. 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 Processor -- start 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- start start 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 0x7f9c24198ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24100d60 0x7f9c241993e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:52.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c241a09d0 con 0x7f9c24101700 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c241a0b10 con 0x7f9c24100d60 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.418+0000 7f9c2977e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c241a0ca0 con 0x7f9c240ffb70 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51804/0 (socket says 172.21.15.90:51804) 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 -- 172.21.15.90:0/1422082500 learned_addr learned my addr 172.21.15.90:0/1422082500 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c22ffd700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 0x7f9c24198ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 -- 172.21.15.90:0/1422082500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 msgr2=0x7f9c24198ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 0x7f9c24198ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c227fc700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24100d60 0x7f9c241993e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 -- 172.21.15.90:0/1422082500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24100d60 msgr2=0x7f9c241993e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24100d60 0x7f9c241993e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 -- 172.21.15.90:0/1422082500 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c140095f0 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c237fe700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9c1c00b970 tx=0x7f9c1c00bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c1c00c930 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9c241a0f80 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9c1c014440 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c1c0075d0 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.419+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c241a1400 con 0x7f9c24101700 2024-04-06T14:03:52.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.420+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9c24067a20 con 0x7f9c24101700 2024-04-06T14:03:52.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.421+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f9c1c0145b0 con 0x7f9c24101700 2024-04-06T14:03:52.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.422+0000 7f9c0bfff700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 0x7f9c0c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:52.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.422+0000 7f9c22ffd700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 0x7f9c0c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:52.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.422+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f9c1c006e40 con 0x7f9c24101700 2024-04-06T14:03:52.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.422+0000 7f9c22ffd700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 0x7f9c0c0a35e0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f9c24100bc0 tx=0x7f9c1800b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:52.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.425+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9c1c095b60 con 0x7f9c24101700 2024-04-06T14:03:52.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.679+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9c2410b450 con 0x7f9c24101700 2024-04-06T14:03:52.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.680+0000 7f9c0bfff700 1 -- 172.21.15.90:0/1422082500 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) v1 ==== 94+0+8088 (secure 0 0 0) 0x7f9c1c099180 con 0x7f9c24101700 2024-04-06T14:03:52.681 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:52.681 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24457,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24469,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24485,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24490,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2024-04-06T14:03:52.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.682+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 msgr2=0x7f9c0c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:52.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.682+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 0x7f9c0c0a35e0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f9c24100bc0 tx=0x7f9c1800b410 comp rx=0 tx=0).stop 2024-04-06T14:03:52.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.682+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 msgr2=0x7f9c241a0400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.682+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9c1c00b970 tx=0x7f9c1c00bc80 comp rx=0 tx=0).stop 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 shutdown_connections 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c0c0a1120 0x7f9c0c0a35e0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c24101700 0x7f9c241a0400 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c240ffb70 0x7f9c24198ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 --2- 172.21.15.90:0/1422082500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c24100d60 0x7f9c241993e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 >> 172.21.15.90:0/1422082500 conn(0x7f9c240fb100 msgr2=0x7f9c240fcc10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 shutdown_connections 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:52.683+0000 7f9c2977e700 1 -- 172.21.15.90:0/1422082500 wait complete. 2024-04-06T14:03:52.684 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 11 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: Deploying daemon mds.a on smithi090 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.90:6838/1327801998,v1:172.21.15.90:6839/1327801998] up:boot 2024-04-06T14:03:53.224 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: fsmap 10 up:standby 2024-04-06T14:03:53.225 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:03:53.225 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1422082500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:53.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: Deploying daemon mds.a on smithi090 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.90:6838/1327801998,v1:172.21.15.90:6839/1327801998] up:boot 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: fsmap 10 up:standby 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:03:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1422082500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: Deploying daemon mds.a on smithi090 2024-04-06T14:03:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:53.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.90:6838/1327801998,v1:172.21.15.90:6839/1327801998] up:boot 2024-04-06T14:03:53.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: fsmap 10 up:standby 2024-04-06T14:03:53.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:03:53.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1422082500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:53.488 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:03:53 smithi090 systemd[1]: Starting Ceph mds.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:54.038 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:03:53 smithi090 ceph-mds[95222]: -- 172.21.15.90:0/1872812351 <== mon.0 v2:172.21.15.90:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5597ca51c1a0 con 0x5597ca4f2c00 2024-04-06T14:03:54.038 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:03:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-a[95199]: starting mds.a at 2024-04-06T14:03:54.038 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:03:53 smithi090 bash[95184]: 9cd742b0e160f4c257ebcc53a4a1f54730471be00c9747c441d2bea1531b9a03 2024-04-06T14:03:54.038 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:03:53 smithi090 systemd[1]: Started Ceph mds.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:54.697 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:54.788 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:54.788 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:54.788 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:54.788 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:54.788 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: Deploying daemon mds.b on smithi105 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.90:6840/761413365,v1:172.21.15.90:6841/761413365] up:boot 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: fsmap 11 up:standby 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:03:54.789 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:54 smithi105 ceph-mon[46969]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:54.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.838+0000 7fe58c238700 1 -- 172.21.15.90:0/4117360729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5840ff6a0 msgr2=0x7fe584109270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.838+0000 7fe58c238700 1 --2- 172.21.15.90:0/4117360729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5840ff6a0 0x7fe584109270 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fe57c00b0d0 tx=0x7fe57c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- 172.21.15.90:0/4117360729 shutdown_connections 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- 172.21.15.90:0/4117360729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5840ff6a0 0x7fe584109270 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- 172.21.15.90:0/4117360729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe584102290 0x7fe5841026b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- 172.21.15.90:0/4117360729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 0x7fe5840ff160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- 172.21.15.90:0/4117360729 >> 172.21.15.90:0/4117360729 conn(0x7fe5840fa8c0 msgr2=0x7fe5840fcce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- 172.21.15.90:0/4117360729 shutdown_connections 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- 172.21.15.90:0/4117360729 wait complete. 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 Processor -- start 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- start start 2024-04-06T14:03:54.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 0x7fe584101940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5840ff6a0 0x7fe5840fffd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe584100cb0 con 0x7fe584102290 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe584100e20 con 0x7fe5840ff6a0 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.839+0000 7fe58c238700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5841a0d70 con 0x7fe5840fed00 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54340/0 (socket says 172.21.15.90:54340) 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 -- 172.21.15.90:0/2066750990 learned_addr learned my addr 172.21.15.90:0/2066750990 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe5897d3700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5840ff6a0 0x7fe5840fffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe589fd4700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 0x7fe584101940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 -- 172.21.15.90:0/2066750990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 msgr2=0x7fe584101940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 0x7fe584101940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 -- 172.21.15.90:0/2066750990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5840ff6a0 msgr2=0x7fe5840fffd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5840ff6a0 0x7fe5840fffd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 -- 172.21.15.90:0/2066750990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe57c009d00 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58a7d5700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fe57c007bd0 tx=0x7fe57c009440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe57c00e040 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe57c012dc0 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5841a0fa0 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe57c01b450 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.840+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5841a13c0 con 0x7fe584102290 2024-04-06T14:03:54.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.841+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5841081e0 con 0x7fe584102290 2024-04-06T14:03:54.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.842+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fe57c012900 con 0x7fe584102290 2024-04-06T14:03:54.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.843+0000 7fe57affd700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 0x7fe5700a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:54.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.843+0000 7fe589fd4700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 0x7fe5700a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:54.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.843+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fe57c0ce160 con 0x7fe584102290 2024-04-06T14:03:54.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.843+0000 7fe589fd4700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 0x7fe5700a3660 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fe574005950 tx=0x7fe5740058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:54.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:54.847+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe57c097e20 con 0x7fe584102290 2024-04-06T14:03:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: Deploying daemon mds.b on smithi105 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.90:6840/761413365,v1:172.21.15.90:6841/761413365] up:boot 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: fsmap 11 up:standby 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:54 smithi090 ceph-mon[43420]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:55.039 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:03:55 smithi105 systemd[1]: Starting Ceph mds.b for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:03:55.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.088+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe584067e70 con 0x7fe584102290 2024-04-06T14:03:55.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.089+0000 7fe57affd700 1 -- 172.21.15.90:0/2066750990 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) v1 ==== 94+0+8843 (secure 0 0 0) 0x7fe57c09b440 con 0x7fe584102290 2024-04-06T14:03:55.089 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:55.090 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24457,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24469,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24485,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24490,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 msgr2=0x7fe5700a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 0x7fe5700a3660 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fe574005950 tx=0x7fe5740058e0 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 msgr2=0x7fe584100510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fe57c007bd0 tx=0x7fe57c009440 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 shutdown_connections 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5700a11a0 0x7fe5700a3660 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe584102290 0x7fe584100510 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5840fed00 0x7fe584101940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 --2- 172.21.15.90:0/2066750990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5840ff6a0 0x7fe5840fffd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 >> 172.21.15.90:0/2066750990 conn(0x7fe5840fa8c0 msgr2=0x7fe584105320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 shutdown_connections 2024-04-06T14:03:55.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:55.091+0000 7fe58c238700 1 -- 172.21.15.90:0/2066750990 wait complete. 2024-04-06T14:03:55.093 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 12 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: Deploying daemon mds.b on smithi105 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.90:6840/761413365,v1:172.21.15.90:6841/761413365] up:boot 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: fsmap 11 up:standby 2024-04-06T14:03:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:03:55.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:54 smithi134 ceph-mon[47549]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:55.747 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mds[84441]: -- 172.21.15.105:0/3265655385 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558a590ea1a0 con 0x558a590c0c00 2024-04-06T14:03:55.747 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: starting mds.b at 2024-04-06T14:03:55.747 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:03:55 smithi105 bash[84403]: 5abc943e0837db87206e4a116954d965c31078d0515c8998cb5b53422abc1eb2 2024-04-06T14:03:55.747 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:03:55 smithi105 systemd[1]: Started Ceph mds.b for 949dca1a-f41d-11ee-b649-cb9ed24678a4. 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2066750990' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:55 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2066750990' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:55 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2066750990' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:55 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:57.104 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 -- 172.21.15.90:0/1554846175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c0fef60 msgr2=0x7f731c0ff380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 --2- 172.21.15.90:0/1554846175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c0fef60 0x7f731c0ff380 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f7310009990 tx=0x7f7310009ca0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 -- 172.21.15.90:0/1554846175 shutdown_connections 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 --2- 172.21.15.90:0/1554846175 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c1028d0 0x7f731c109180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 --2- 172.21.15.90:0/1554846175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c0fef60 0x7f731c0ff380 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 --2- 172.21.15.90:0/1554846175 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c101f30 0x7f731c102390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 -- 172.21.15.90:0/1554846175 >> 172.21.15.90:0/1554846175 conn(0x7f731c0fa890 msgr2=0x7f731c0fccd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 -- 172.21.15.90:0/1554846175 shutdown_connections 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.236+0000 7f7321508700 1 -- 172.21.15.90:0/1554846175 wait complete. 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 Processor -- start 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 -- start start 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c0fef60 0x7f731c196be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c1028d0 0x7f731c19e140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f731c19e710 con 0x7f731c101f30 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f731c19e850 con 0x7f731c0fef60 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f7321508700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f731c19e9e0 con 0x7f731c1028d0 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f731a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f731a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54354/0 (socket says 172.21.15.90:54354) 2024-04-06T14:03:57.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f731a7fc700 1 -- 172.21.15.90:0/529490892 learned_addr learned my addr 172.21.15.90:0/529490892 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f731affd700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c0fef60 0x7f731c196be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.237+0000 7f731b7fe700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c1028d0 0x7f731c19e140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 -- 172.21.15.90:0/529490892 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c1028d0 msgr2=0x7f731c19e140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c1028d0 0x7f731c19e140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 -- 172.21.15.90:0/529490892 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c0fef60 msgr2=0x7f731c196be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c0fef60 0x7f731c196be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 -- 172.21.15.90:0/529490892 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73100095f0 con 0x7f731c101f30 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f731a7fc700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f730c00d950 tx=0x7f730c00dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f730c00f890 con 0x7f731c101f30 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f730c010460 con 0x7f731c101f30 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f731c19ecc0 con 0x7f731c101f30 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f730c00e610 con 0x7f731c101f30 2024-04-06T14:03:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.238+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f731c19f1d0 con 0x7f731c101f30 2024-04-06T14:03:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.239+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f731c108160 con 0x7f731c101f30 2024-04-06T14:03:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.240+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f730c0108d0 con 0x7f731c101f30 2024-04-06T14:03:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.240+0000 7f7303fff700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 0x7f73040a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.240+0000 7f731affd700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 0x7f73040a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.241+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f730c0cc1f0 con 0x7f731c101f30 2024-04-06T14:03:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.241+0000 7f731affd700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 0x7f73040a35e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f731000b680 tx=0x7f7310005fd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:57.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.245+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f730c095e20 con 0x7f731c101f30 2024-04-06T14:03:57.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:boot 2024-04-06T14:03:57.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: fsmap 12 up:standby 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:56 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:boot 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: fsmap 12 up:standby 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:56 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:boot 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: fsmap 12 up:standby 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:56 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:57.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.501+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f731c067e70 con 0x7f731c101f30 2024-04-06T14:03:57.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.501+0000 7f7303fff700 1 -- 172.21.15.90:0/529490892 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) v1 ==== 94+0+9604 (secure 0 0 0) 0x7f730c099440 con 0x7f731c101f30 2024-04-06T14:03:57.502 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:57.502 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24457,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24458,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24469,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24485,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24490,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24515,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":13}],"filesystems":[]} 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 msgr2=0x7f73040a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 0x7f73040a35e0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f731000b680 tx=0x7f7310005fd0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 msgr2=0x7f731c197120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f730c00d950 tx=0x7f730c00dc60 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 shutdown_connections 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f73040a1120 0x7f73040a35e0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f731c1028d0 0x7f731c19e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f731c0fef60 0x7f731c196be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 --2- 172.21.15.90:0/529490892 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f731c101f30 0x7f731c197120 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 >> 172.21.15.90:0/529490892 conn(0x7f731c0fa890 msgr2=0x7f731c1052a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 shutdown_connections 2024-04-06T14:03:57.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.504+0000 7f7321508700 1 -- 172.21.15.90:0/529490892 wait complete. 2024-04-06T14:03:57.506 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 13 2024-04-06T14:03:57.518 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:03:57.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 -- 172.21.15.90:0/1496841626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980101700 msgr2=0x7f6980107fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 --2- 172.21.15.90:0/1496841626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980101700 0x7f6980107fb0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f697800b0d0 tx=0x7f697800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 -- 172.21.15.90:0/1496841626 shutdown_connections 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 --2- 172.21.15.90:0/1496841626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980101700 0x7f6980107fb0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 --2- 172.21.15.90:0/1496841626 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69800ffb70 0x7f69800fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 --2- 172.21.15.90:0/1496841626 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6980100d60 0x7f69801011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.666+0000 7f69869cc700 1 -- 172.21.15.90:0/1496841626 >> 172.21.15.90:0/1496841626 conn(0x7f69800fb100 msgr2=0x7f69800fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- 172.21.15.90:0/1496841626 shutdown_connections 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- 172.21.15.90:0/1496841626 wait complete. 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 Processor -- start 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- start start 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 0x7f6980195be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6980101700 0x7f698019d140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f698019d710 con 0x7f6980100d60 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f698019d850 con 0x7f69800ffb70 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.667+0000 7f69869cc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f698019d9e0 con 0x7f6980101700 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 0x7f6980195be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54372/0 (socket says 172.21.15.90:54372) 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 -- 172.21.15.90:0/2766430375 learned_addr learned my addr 172.21.15.90:0/2766430375 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f6984f69700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6980101700 0x7f698019d140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 -- 172.21.15.90:0/2766430375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6980101700 msgr2=0x7f698019d140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6980101700 0x7f698019d140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 -- 172.21.15.90:0/2766430375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 msgr2=0x7f6980195be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 0x7f6980195be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 -- 172.21.15.90:0/2766430375 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6978009d00 con 0x7f6980100d60 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697ffff700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 0x7f6980195be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:03:57.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697f7fe700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f6974009970 tx=0x7f6974009c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:57.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6974012070 con 0x7f6980100d60 2024-04-06T14:03:57.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f697400d260 con 0x7f6980100d60 2024-04-06T14:03:57.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6980067e60 con 0x7f6980100d60 2024-04-06T14:03:57.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6974017860 con 0x7f6980100d60 2024-04-06T14:03:57.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.668+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6980068370 con 0x7f6980100d60 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.669+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6980067a20 con 0x7f6980100d60 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.671+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f6974007af0 con 0x7f6980100d60 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.672+0000 7f697d7fa700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 0x7f69680a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.672+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f69740cd680 con 0x7f6980100d60 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.672+0000 7f697ffff700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 0x7f69680a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:57.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.672+0000 7f697ffff700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 0x7f69680a35e0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f6980100bc0 tx=0x7f697000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:57.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.675+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f69740972b0 con 0x7f6980100d60 2024-04-06T14:03:57.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.899+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f698010b560 con 0x7f6980100d60 2024-04-06T14:03:57.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.900+0000 7f697d7fa700 1 -- 172.21.15.90:0/2766430375 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v77) v1 ==== 74+0+18411 (secure 0 0 0) 0x7f697409a8d0 con 0x7f6980100d60 2024-04-06T14:03:57.901 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:57.901 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":77,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:03:26.970706+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":72,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:03:57.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 msgr2=0x7f69680a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 0x7f69680a35e0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f6980100bc0 tx=0x7f697000b410 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 msgr2=0x7f6980196120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f6974009970 tx=0x7f6974009c80 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 shutdown_connections 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f69680a1120 0x7f69680a35e0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6980101700 0x7f698019d140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69800ffb70 0x7f6980195be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 --2- 172.21.15.90:0/2766430375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6980100d60 0x7f6980196120 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 >> 172.21.15.90:0/2766430375 conn(0x7f69800fb100 msgr2=0x7f6980104060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 shutdown_connections 2024-04-06T14:03:57.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:57.903+0000 7f69869cc700 1 -- 172.21.15.90:0/2766430375 wait complete. 2024-04-06T14:03:57.917 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:03:57.918 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 -- 172.21.15.90:0/723430504 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 msgr2=0x7fc91c1027a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 --2- 172.21.15.90:0/723430504 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c1027a0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fc90c0099c0 tx=0x7fc90c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 -- 172.21.15.90:0/723430504 shutdown_connections 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 --2- 172.21.15.90:0/723430504 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0ff790 0x7fc91c109360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 --2- 172.21.15.90:0/723430504 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c1027a0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 --2- 172.21.15.90:0/723430504 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0fedf0 0x7fc91c0ff250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.049+0000 7fc921c9b700 1 -- 172.21.15.90:0/723430504 >> 172.21.15.90:0/723430504 conn(0x7fc91c0fa990 msgr2=0x7fc91c0fcdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- 172.21.15.90:0/723430504 shutdown_connections 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- 172.21.15.90:0/723430504 wait complete. 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 Processor -- start 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- start start 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 0x7fc91c101a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc91c100da0 con 0x7fc91c102380 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc91c100f10 con 0x7fc91c0fedf0 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.050+0000 7fc921c9b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc91c1a0df0 con 0x7fc91c0ff790 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 0x7fc91c101a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54384/0 (socket says 172.21.15.90:54384) 2024-04-06T14:03:58.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:47900/0 (socket says 172.21.15.90:47900) 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 -- 172.21.15.90:0/1035432369 learned_addr learned my addr 172.21.15.90:0/1035432369 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 -- 172.21.15.90:0/1035432369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 msgr2=0x7fc91c1000c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 -- 172.21.15.90:0/1035432369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 msgr2=0x7fc91c101a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 0x7fc91c101a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 -- 172.21.15.90:0/1035432369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc910009520 con 0x7fc91c102380 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91b7fe700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 0x7fc91c101a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91affd700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc91bfff700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fc91400ef30 tx=0x7fc91400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc914018070 con 0x7fc91c102380 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc90c0095f0 con 0x7fc91c102380 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc91400f040 con 0x7fc91c102380 2024-04-06T14:03:58.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc91400cca0 con 0x7fc91c102380 2024-04-06T14:03:58.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.051+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc91c1a1350 con 0x7fc91c102380 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.052+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc91c108440 con 0x7fc91c102380 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.053+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fc914003a00 con 0x7fc91c102380 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.054+0000 7fc918ff9700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 0x7fc9040a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.054+0000 7fc91b7fe700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 0x7fc9040a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.054+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fc9140cca70 con 0x7fc91c102380 2024-04-06T14:03:58.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.055+0000 7fc91b7fe700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 0x7fc9040a35e0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc90c009990 tx=0x7fc90c01a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.058+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc914096690 con 0x7fc91c102380 2024-04-06T14:03:58.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:58.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:58.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:58.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/529490892' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:57 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2766430375' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.283+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fc91c107e20 con 0x7fc91c102380 2024-04-06T14:03:58.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.283+0000 7fc918ff9700 1 -- 172.21.15.90:0/1035432369 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v77) v1 ==== 93+0+10 (secure 0 0 0) 0x7fc914099cb0 con 0x7fc91c102380 2024-04-06T14:03:58.284 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 msgr2=0x7fc9040a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 0x7fc9040a35e0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fc90c009990 tx=0x7fc90c01a040 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 msgr2=0x7fc91c100600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fc91400ef30 tx=0x7fc91400c3a0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 shutdown_connections 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc9040a1120 0x7fc9040a35e0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc91c102380 0x7fc91c100600 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc91c0fedf0 0x7fc91c101a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 --2- 172.21.15.90:0/1035432369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc91c0ff790 0x7fc91c1000c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 >> 172.21.15.90:0/1035432369 conn(0x7fc91c0fa990 msgr2=0x7fc91c105580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 shutdown_connections 2024-04-06T14:03:58.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.286+0000 7fc921c9b700 1 -- 172.21.15.90:0/1035432369 wait complete. 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/529490892' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:57 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2766430375' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.300 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/529490892' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:57 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2766430375' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- 172.21.15.90:0/2219933108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 msgr2=0x7f5dc81022e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2219933108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc81022e0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f5dbc0099c0 tx=0x7f5dbc009cd0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- 172.21.15.90:0/2219933108 shutdown_connections 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2219933108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8102820 0x7f5dc81090d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2219933108 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8100c90 0x7f5dc81010b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2219933108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc81022e0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- 172.21.15.90:0/2219933108 >> 172.21.15.90:0/2219933108 conn(0x7f5dc80fc220 msgr2=0x7f5dc80fe640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- 172.21.15.90:0/2219933108 shutdown_connections 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- 172.21.15.90:0/2219933108 wait complete. 2024-04-06T14:03:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 Processor -- start 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.449+0000 7f5dce30e700 1 -- start start 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8100c90 0x7f5dc8198df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 0x7f5dc81a0310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5dc81a08e0 con 0x7f5dc8101e80 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5dc81a0a50 con 0x7f5dc8100c90 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dce30e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5dc81a0c10 con 0x7f5dc8102820 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54408/0 (socket says 172.21.15.90:54408) 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 -- 172.21.15.90:0/2265478285 learned_addr learned my addr 172.21.15.90:0/2265478285 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc7fff700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8100c90 0x7f5dc8198df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dcc8ab700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 0x7f5dc81a0310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 -- 172.21.15.90:0/2265478285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 msgr2=0x7f5dc81a0310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 0x7f5dc81a0310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 -- 172.21.15.90:0/2265478285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8100c90 msgr2=0x7f5dc8198df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8100c90 0x7f5dc8198df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 -- 172.21.15.90:0/2265478285 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5dbc0095f0 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dcc8ab700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 0x7f5dc81a0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc77fe700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f5dbc003730 tx=0x7f5dbc00fa00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.450+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5dbc01d070 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.451+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5dc81a0e90 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.451+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5dbc00fe70 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.451+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5dbc017650 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.451+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5dc81a1250 con 0x7f5dc8101e80 2024-04-06T14:03:58.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.451+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5dc8067a20 con 0x7f5dc8101e80 2024-04-06T14:03:58.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.453+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f5dbc022470 con 0x7f5dc8101e80 2024-04-06T14:03:58.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.454+0000 7f5dc57fa700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 0x7f5db00a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.454+0000 7f5dc7fff700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 0x7f5db00a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.454+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f5dbc0cdb80 con 0x7f5dc8101e80 2024-04-06T14:03:58.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.454+0000 7f5dc7fff700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 0x7f5db00a35e0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f5dc819a270 tx=0x7f5db8009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.457+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5dbc0977a0 con 0x7f5dc8101e80 2024-04-06T14:03:58.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.666+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7f5dc810c740 con 0x7f5dc8101e80 2024-04-06T14:03:58.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.666+0000 7f5dc57fa700 1 -- 172.21.15.90:0/2265478285 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v77) v1 ==== 84+0+65 (secure 0 0 0) 0x7f5dbc09adc0 con 0x7f5dc8101e80 2024-04-06T14:03:58.667 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:03:58.667 INFO:teuthology.orchestra.run.smithi090.stdout:[ 2024-04-06T14:03:58.667 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:03:58.668 INFO:teuthology.orchestra.run.smithi090.stdout: "poolnum": 1, 2024-04-06T14:03:58.668 INFO:teuthology.orchestra.run.smithi090.stdout: "poolname": ".mgr" 2024-04-06T14:03:58.668 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:03:58.668 INFO:teuthology.orchestra.run.smithi090.stdout:] 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.668+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 msgr2=0x7f5db00a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.668+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 0x7f5db00a35e0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f5dc819a270 tx=0x7f5db8009260 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 msgr2=0x7f5dc8199330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f5dbc003730 tx=0x7f5dbc00fa00 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 shutdown_connections 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5db00a1120 0x7f5db00a35e0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5dc8102820 0x7f5dc81a0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5dc8100c90 0x7f5dc8198df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 --2- 172.21.15.90:0/2265478285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5dc8101e80 0x7f5dc8199330 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 >> 172.21.15.90:0/2265478285 conn(0x7f5dc80fc220 msgr2=0x7f5dc8072fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:58.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 shutdown_connections 2024-04-06T14:03:58.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.669+0000 7f5dce30e700 1 -- 172.21.15.90:0/2265478285 wait complete. 2024-04-06T14:03:58.681 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-04-06T14:03:58.681 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 -- 172.21.15.90:0/2387916488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80102910 msgr2=0x7f9e801091c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 --2- 172.21.15.90:0/2387916488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80102910 0x7f9e801091c0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9e7800b0d0 tx=0x7f9e7800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 -- 172.21.15.90:0/2387916488 shutdown_connections 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 --2- 172.21.15.90:0/2387916488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80102910 0x7f9e801091c0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 --2- 172.21.15.90:0/2387916488 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80100d80 0x7f9e801011a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 --2- 172.21.15.90:0/2387916488 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80101f70 0x7f9e801023d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 -- 172.21.15.90:0/2387916488 >> 172.21.15.90:0/2387916488 conn(0x7f9e800fc2f0 msgr2=0x7f9e800fe730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 -- 172.21.15.90:0/2387916488 shutdown_connections 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.808+0000 7f9e883af700 1 -- 172.21.15.90:0/2387916488 wait complete. 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 Processor -- start 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 -- start start 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80101f70 0x7f9e801972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80102910 0x7f9e8019e300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e8019e8d0 con 0x7f9e80100d80 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e8019ea10 con 0x7f9e80101f70 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e883af700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e8019eba0 con 0x7f9e80102910 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54430/0 (socket says 172.21.15.90:54430) 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 -- 172.21.15.90:0/3483840656 learned_addr learned my addr 172.21.15.90:0/3483840656 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8594a700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80101f70 0x7f9e801972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8694c700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80102910 0x7f9e8019e300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 -- 172.21.15.90:0/3483840656 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80102910 msgr2=0x7f9e8019e300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80102910 0x7f9e8019e300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 -- 172.21.15.90:0/3483840656 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80101f70 msgr2=0x7f9e801972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80101f70 0x7f9e801972e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 -- 172.21.15.90:0/3483840656 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9e78009d00 con 0x7f9e80100d80 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e8614b700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9e7000e970 tx=0x7f9e7000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e70009790 con 0x7f9e80100d80 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.809+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9e7000c7e0 con 0x7f9e80100d80 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.810+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e8019ee80 con 0x7f9e80100d80 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.810+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9e70003c70 con 0x7f9e80100d80 2024-04-06T14:03:58.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.810+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e8019f2b0 con 0x7f9e80100d80 2024-04-06T14:03:58.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.810+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e80108130 con 0x7f9e80100d80 2024-04-06T14:03:58.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.811+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f9e7000cc50 con 0x7f9e80100d80 2024-04-06T14:03:58.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.812+0000 7f9e777fe700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 0x7f9e6c0a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:03:58.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.812+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(77..77 src has 1..77) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f9e70014070 con 0x7f9e80100d80 2024-04-06T14:03:58.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.812+0000 7f9e8594a700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 0x7f9e6c0a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:03:58.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.812+0000 7f9e8594a700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 0x7f9e6c0a3660 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f9e7c009c60 tx=0x7f9e7c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:03:58.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:58.815+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9e70096310 con 0x7f9e80100d80 2024-04-06T14:03:59.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:03:59.014+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f9e80067270 con 0x7f9e80100d80 2024-04-06T14:03:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1035432369' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:03:58 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2265478285' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1035432369' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:59.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:59.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:03:58 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2265478285' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1035432369' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:03:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:03:59.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:03:58 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2265478285' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-04-06T14:04:00.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.008+0000 7f9e777fe700 1 -- 172.21.15.90:0/3483840656 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v78) v1 ==== 138+0+0 (secure 0 0 0) 0x7f9e70003dd0 con 0x7f9e80100d80 2024-04-06T14:04:00.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.014+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 msgr2=0x7f9e6c0a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:00.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.014+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 0x7f9e6c0a3660 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f9e7c009c60 tx=0x7f9e7c009260 comp rx=0 tx=0).stop 2024-04-06T14:04:00.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.014+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 msgr2=0x7f9e80196da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:00.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.014+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9e7000e970 tx=0x7f9e7000ec80 comp rx=0 tx=0).stop 2024-04-06T14:04:00.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 shutdown_connections 2024-04-06T14:04:00.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9e6c0a11a0 0x7f9e6c0a3660 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e80102910 0x7f9e8019e300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e80100d80 0x7f9e80196da0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 --2- 172.21.15.90:0/3483840656 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e80101f70 0x7f9e801972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 >> 172.21.15.90:0/3483840656 conn(0x7f9e800fc2f0 msgr2=0x7f9e80105270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 shutdown_connections 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.015+0000 7f9e883af700 1 -- 172.21.15.90:0/3483840656 wait complete. 2024-04-06T14:04:00.016 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'cephfs_metadata' created 2024-04-06T14:04:00.030 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- 172.21.15.90:0/2895048590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 msgr2=0x7fc6a0102480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 --2- 172.21.15.90:0/2895048590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0102480 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fc694009990 tx=0x7fc694009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- 172.21.15.90:0/2895048590 shutdown_connections 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 --2- 172.21.15.90:0/2895048590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a0109270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 --2- 172.21.15.90:0/2895048590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 0x7fc6a00ff470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 --2- 172.21.15.90:0/2895048590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0102480 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- 172.21.15.90:0/2895048590 >> 172.21.15.90:0/2895048590 conn(0x7fc6a00fa980 msgr2=0x7fc6a00fcdc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- 172.21.15.90:0/2895048590 shutdown_connections 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- 172.21.15.90:0/2895048590 wait complete. 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 Processor -- start 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 -- start start 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.165+0000 7fc6a736e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 0x7fc6a0196d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a019e260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6a019e830 con 0x7fc6a0102020 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6a019e9a0 con 0x7fc6a01029c0 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6a019eb60 con 0x7fc6a00ff050 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54450/0 (socket says 172.21.15.90:54450) 2024-04-06T14:04:00.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 -- 172.21.15.90:0/131368936 learned_addr learned my addr 172.21.15.90:0/131368936 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a510a700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 0x7fc6a0196d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a590b700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a019e260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 -- 172.21.15.90:0/131368936 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 msgr2=0x7fc6a0196d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 0x7fc6a0196d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 -- 172.21.15.90:0/131368936 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 msgr2=0x7fc6a019e260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a019e260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 -- 172.21.15.90:0/131368936 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6940095f0 con 0x7fc6a0102020 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a590b700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a019e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a4909700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc6940095c0 tx=0x7fc69400f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc69401d070 con 0x7fc6a0102020 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc694022470 con 0x7fc6a0102020 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6a019ede0 con 0x7fc6a0102020 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc694017650 con 0x7fc6a0102020 2024-04-06T14:04:00.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.166+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6a019f1b0 con 0x7fc6a0102020 2024-04-06T14:04:00.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.167+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc6a01082d0 con 0x7fc6a0102020 2024-04-06T14:04:00.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.168+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fc694022850 con 0x7fc6a0102020 2024-04-06T14:04:00.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.169+0000 7fc69e7fc700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 0x7fc6880a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:00.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.169+0000 7fc6a510a700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 0x7fc6880a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:00.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.169+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(78..78 src has 1..78) v4 ==== 7497+0+0 (secure 0 0 0) 0x7fc6940cde60 con 0x7fc6a0102020 2024-04-06T14:04:00.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.169+0000 7fc6a510a700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 0x7fc6880a3660 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7fc6a00fc5c0 tx=0x7fc69000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:00.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.173+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc694097910 con 0x7fc6a0102020 2024-04-06T14:04:00.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:00 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-06T14:04:00.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:04:00.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:00.372+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7fc6a0067f50 con 0x7fc6a0102020 2024-04-06T14:04:00.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:00 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-06T14:04:00.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:04:00.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:00 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-04-06T14:04:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:04:01.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.016+0000 7fc69e7fc700 1 -- 172.21.15.90:0/131368936 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data' created v79) v1 ==== 170+0+0 (secure 0 0 0) 0x7fc69409af30 con 0x7fc6a0102020 2024-04-06T14:04:01.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.022+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 msgr2=0x7fc6880a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.022+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 0x7fc6880a3660 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7fc6a00fc5c0 tx=0x7fc69000b410 comp rx=0 tx=0).stop 2024-04-06T14:04:01.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.022+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 msgr2=0x7fc6a0197250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.022+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc6940095c0 tx=0x7fc69400f690 comp rx=0 tx=0).stop 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 shutdown_connections 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc6880a11a0 0x7fc6880a3660 secure :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7fc6a00fc5c0 tx=0x7fc69000b410 comp rx=0 tx=0).stop 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc6a01029c0 0x7fc6a019e260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc6a00ff050 0x7fc6a0196d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 --2- 172.21.15.90:0/131368936 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc6a0102020 0x7fc6a0197250 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.023+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 >> 172.21.15.90:0/131368936 conn(0x7fc6a00fa980 msgr2=0x7fc6a0105410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.026+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 shutdown_connections 2024-04-06T14:04:01.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.026+0000 7fc6a736e700 1 -- 172.21.15.90:0/131368936 wait complete. 2024-04-06T14:04:01.031 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'cephfs_data' created 2024-04-06T14:04:01.047 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2024-04-06T14:04:01.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 -- 172.21.15.90:0/4115497833 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1600ffc70 msgr2=0x7ff160100090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 --2- 172.21.15.90:0/4115497833 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1600ffc70 0x7ff160100090 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7ff1540099c0 tx=0x7ff154009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 -- 172.21.15.90:0/4115497833 shutdown_connections 2024-04-06T14:04:01.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 --2- 172.21.15.90:0/4115497833 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff160101800 0x7ff1601080b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 --2- 172.21.15.90:0/4115497833 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1600ffc70 0x7ff160100090 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 --2- 172.21.15.90:0/4115497833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160100e60 0x7ff1601012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 -- 172.21.15.90:0/4115497833 >> 172.21.15.90:0/4115497833 conn(0x7ff1600fb200 msgr2=0x7ff1600fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 -- 172.21.15.90:0/4115497833 shutdown_connections 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.174+0000 7ff1683ce700 1 -- 172.21.15.90:0/4115497833 wait complete. 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 Processor -- start 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 -- start start 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1600ffc70 0x7ff160196db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160101800 0x7ff16019e310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff16019e8e0 con 0x7ff160100e60 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff16019ea20 con 0x7ff160101800 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff1683ce700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff16019ebb0 con 0x7ff1600ffc70 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54468/0 (socket says 172.21.15.90:54468) 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 -- 172.21.15.90:0/3785683434 learned_addr learned my addr 172.21.15.90:0/3785683434 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff16616a700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1600ffc70 0x7ff160196db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff16696b700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160101800 0x7ff16019e310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 -- 172.21.15.90:0/3785683434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1600ffc70 msgr2=0x7ff160196db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1600ffc70 0x7ff160196db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 -- 172.21.15.90:0/3785683434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160101800 msgr2=0x7ff16019e310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160101800 0x7ff16019e310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.175+0000 7ff165969700 1 -- 172.21.15.90:0/3785683434 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1540095f0 con 0x7ff160100e60 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff165969700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7ff15c00d950 tx=0x7ff15c00dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff15c00f890 con 0x7ff160100e60 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff15c010460 con 0x7ff160100e60 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff16019ee90 con 0x7ff160100e60 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff15c00e610 con 0x7ff160100e60 2024-04-06T14:04:01.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.176+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff16019f310 con 0x7ff160100e60 2024-04-06T14:04:01.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.177+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff160067b00 con 0x7ff160100e60 2024-04-06T14:04:01.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.178+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7ff15c0108d0 con 0x7ff160100e60 2024-04-06T14:04:01.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.179+0000 7ff1537fe700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 0x7ff14c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.179+0000 7ff16616a700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 0x7ff14c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.179+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(79..79 src has 1..79) v4 ==== 7879+0+0 (secure 0 0 0) 0x7ff15c0cc4b0 con 0x7ff160100e60 2024-04-06T14:04:01.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.179+0000 7ff16616a700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 0x7ff14c0a35e0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7ff154009f50 tx=0x7ff154005ea0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.183+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff15c095df0 con 0x7ff160100e60 2024-04-06T14:04:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:01 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-06T14:04:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:01 smithi090 ceph-mon[43420]: osdmap e78: 12 total, 12 up, 12 in 2024-04-06T14:04:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:01 smithi090 ceph-mon[43420]: pgmap v216: 33 pgs: 1 creating+peering, 1 active+clean, 31 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:04:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:01 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-06T14:04:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:01 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-06T14:04:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:01 smithi134 ceph-mon[47549]: osdmap e78: 12 total, 12 up, 12 in 2024-04-06T14:04:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:01 smithi134 ceph-mon[47549]: pgmap v216: 33 pgs: 1 creating+peering, 1 active+clean, 31 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:04:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:01 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-06T14:04:01.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.429+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7ff16010b5c0 con 0x7ff160100e60 2024-04-06T14:04:01.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.436+0000 7ff1537fe700 1 -- 172.21.15.90:0/3785683434 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) v1 ==== 172+0+0 (secure 0 0 0) 0x7ff15c0160c0 con 0x7ff160100e60 2024-04-06T14:04:01.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.447+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 msgr2=0x7ff14c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.447+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 0x7ff14c0a35e0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7ff154009f50 tx=0x7ff154005ea0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.447+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 msgr2=0x7ff1601972f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.447+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7ff15c00d950 tx=0x7ff15c00dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:01.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 shutdown_connections 2024-04-06T14:04:01.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff14c0a1120 0x7ff14c0a35e0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff160101800 0x7ff16019e310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1600ffc70 0x7ff160196db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 --2- 172.21.15.90:0/3785683434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff160100e60 0x7ff1601972f0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.450+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 >> 172.21.15.90:0/3785683434 conn(0x7ff1600fb200 msgr2=0x7ff160067e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.453+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 shutdown_connections 2024-04-06T14:04:01.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.454+0000 7ff1683ce700 1 -- 172.21.15.90:0/3785683434 wait complete. 2024-04-06T14:04:01.454 INFO:teuthology.orchestra.run.smithi090.stderr:new fs with metadata pool 2 and data pool 3 2024-04-06T14:04:01.471 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:01.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:01 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3483840656' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-04-06T14:04:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:01 smithi105 ceph-mon[46969]: osdmap e78: 12 total, 12 up, 12 in 2024-04-06T14:04:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:01 smithi105 ceph-mon[46969]: pgmap v216: 33 pgs: 1 creating+peering, 1 active+clean, 31 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:04:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:01 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 -- 172.21.15.90:0/4109186102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 msgr2=0x7f951c1011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 --2- 172.21.15.90:0/4109186102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c1011c0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f95040099c0 tx=0x7f9504009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 -- 172.21.15.90:0/4109186102 shutdown_connections 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 --2- 172.21.15.90:0/4109186102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 --2- 172.21.15.90:0/4109186102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 0x7f951c0fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 --2- 172.21.15.90:0/4109186102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c1011c0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 -- 172.21.15.90:0/4109186102 >> 172.21.15.90:0/4109186102 conn(0x7f951c0fb100 msgr2=0x7f951c0fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 -- 172.21.15.90:0/4109186102 shutdown_connections 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 -- 172.21.15.90:0/4109186102 wait complete. 2024-04-06T14:04:01.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.592+0000 7f95219f5700 1 Processor -- start 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 -- start start 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 0x7f951c068940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c19deb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f951c19e6a0 con 0x7f951c100d60 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f951c19e810 con 0x7f951c0ffb70 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f95219f5700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f951c19e9d0 con 0x7f951c101700 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54488/0 (socket says 172.21.15.90:54488) 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 -- 172.21.15.90:0/487441945 learned_addr learned my addr 172.21.15.90:0/487441945 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951affd700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 0x7f951c068940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951b7fe700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c19deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 -- 172.21.15.90:0/487441945 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 msgr2=0x7f951c19deb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c19deb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 -- 172.21.15.90:0/487441945 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 msgr2=0x7f951c068940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 0x7f951c068940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 -- 172.21.15.90:0/487441945 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95040095f0 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951b7fe700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c19deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.593+0000 7f951a7fc700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f950400b580 tx=0x7f9504009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f950401d070 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9504004010 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f951c19ebf0 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9504022d10 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f951c19f0a0 con 0x7f951c100d60 2024-04-06T14:04:01.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.594+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f951c106f20 con 0x7f951c100d60 2024-04-06T14:04:01.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.596+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f9504022470 con 0x7f951c100d60 2024-04-06T14:04:01.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.596+0000 7f95209f3700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 0x7f95080a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.597+0000 7f951affd700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 0x7f95080a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.597+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f95040ce2f0 con 0x7f951c100d60 2024-04-06T14:04:01.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.597+0000 7f951affd700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 0x7f95080a35e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f951c100bc0 tx=0x7f950c0091c0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.600+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9504097c60 con 0x7f951c100d60 2024-04-06T14:04:01.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.797+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f951c10b380 con 0x7f951c100d60 2024-04-06T14:04:01.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.799+0000 7f95209f3700 1 -- 172.21.15.90:0/487441945 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) v1 ==== 74+0+22121 (secure 0 0 0) 0x7f9504086090 con 0x7f951c100d60 2024-04-06T14:04:01.799 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:01.799 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":80,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:01.430136+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 msgr2=0x7f95080a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 0x7f95080a35e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f951c100bc0 tx=0x7f950c0091c0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 msgr2=0x7f951c19b890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f950400b580 tx=0x7f9504009580 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 shutdown_connections 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95080a1120 0x7f95080a35e0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f951c101700 0x7f951c19deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f951c0ffb70 0x7f951c068940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 --2- 172.21.15.90:0/487441945 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f951c100d60 0x7f951c19b890 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 >> 172.21.15.90:0/487441945 conn(0x7f951c0fb100 msgr2=0x7f951c104060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 shutdown_connections 2024-04-06T14:04:01.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.801+0000 7f95219f5700 1 -- 172.21.15.90:0/487441945 wait complete. 2024-04-06T14:04:01.813 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:01.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- 172.21.15.90:0/300126662 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 msgr2=0x7fa6900fffa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- 172.21.15.90:0/300126662 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa6900fffa0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fa68c0099c0 tx=0x7fa68c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- 172.21.15.90:0/300126662 shutdown_connections 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- 172.21.15.90:0/300126662 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690101710 0x7fa690107fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- 172.21.15.90:0/300126662 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa6900fffa0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- 172.21.15.90:0/300126662 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690100d70 0x7fa6901011d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- 172.21.15.90:0/300126662 >> 172.21.15.90:0/300126662 conn(0x7fa6900fb0f0 msgr2=0x7fa6900fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- 172.21.15.90:0/300126662 shutdown_connections 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- 172.21.15.90:0/300126662 wait complete. 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 Processor -- start 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- start start 2024-04-06T14:04:01.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690100d70 0x7fa69019b930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690101710 0x7fa69019df50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa69019e740 con 0x7fa6900ffb80 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa69019e8b0 con 0x7fa690100d70 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.936+0000 7fa698784700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa69019ea70 con 0x7fa690101710 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54512/0 (socket says 172.21.15.90:54512) 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 -- 172.21.15.90:0/1205501394 learned_addr learned my addr 172.21.15.90:0/1205501394 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696d21700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690101710 0x7fa69019df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa695d1f700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690100d70 0x7fa69019b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 -- 172.21.15.90:0/1205501394 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690101710 msgr2=0x7fa69019df50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690101710 0x7fa69019df50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 -- 172.21.15.90:0/1205501394 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690100d70 msgr2=0x7fa69019b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:01.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690100d70 0x7fa69019b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 -- 172.21.15.90:0/1205501394 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa68c0095f0 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa696520700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fa68c009fd0 tx=0x7fa68c005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa68c01d070 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa68c0059a0 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa69019ec90 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa68c017650 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.937+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa69019f190 con 0x7fa6900ffb80 2024-04-06T14:04:01.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.938+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6901070a0 con 0x7fa6900ffb80 2024-04-06T14:04:01.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.939+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fa68c022470 con 0x7fa6900ffb80 2024-04-06T14:04:01.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.940+0000 7fa6877fe700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 0x7fa67c0a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:01.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.940+0000 7fa695d1f700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 0x7fa67c0a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:01.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.940+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fa68c0ce2d0 con 0x7fa6900ffb80 2024-04-06T14:04:01.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.940+0000 7fa695d1f700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 0x7fa67c0a3660 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa680005950 tx=0x7fa6800058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:01.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:01.943+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fa68c097c40 con 0x7fa6900ffb80 2024-04-06T14:04:02.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.140+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa69010b390 con 0x7fa6900ffb80 2024-04-06T14:04:02.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.142+0000 7fa6877fe700 1 -- 172.21.15.90:0/1205501394 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) v1 ==== 74+0+22121 (secure 0 0 0) 0x7fa68c027090 con 0x7fa6900ffb80 2024-04-06T14:04:02.142 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:02.142 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":80,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:01.430136+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 msgr2=0x7fa67c0a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 0x7fa67c0a3660 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7fa680005950 tx=0x7fa6800058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 msgr2=0x7fa690069ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fa68c009fd0 tx=0x7fa68c005f10 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 shutdown_connections 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fa67c0a11a0 0x7fa67c0a3660 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa690101710 0x7fa69019df50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa6900ffb80 0x7fa690069ba0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 --2- 172.21.15.90:0/1205501394 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa690100d70 0x7fa69019b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 >> 172.21.15.90:0/1205501394 conn(0x7fa6900fb0f0 msgr2=0x7fa6901041e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 shutdown_connections 2024-04-06T14:04:02.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.144+0000 7fa698784700 1 -- 172.21.15.90:0/1205501394 wait complete. 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: osdmap e79: 12 total, 12 up, 12 in 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: osdmap e80: 12 total, 12 up, 12 in 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-06T14:04:02.158 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/487441945' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:02.160 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 -- 172.21.15.90:0/265254317 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0102920 msgr2=0x7f90d01091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 --2- 172.21.15.90:0/265254317 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0102920 0x7f90d01091d0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f90c800b0d0 tx=0x7f90c800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 -- 172.21.15.90:0/265254317 shutdown_connections 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 --2- 172.21.15.90:0/265254317 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0102920 0x7f90d01091d0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 --2- 172.21.15.90:0/265254317 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0100d90 0x7f90d01011b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 --2- 172.21.15.90:0/265254317 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0101f80 0x7f90d01023e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 -- 172.21.15.90:0/265254317 >> 172.21.15.90:0/265254317 conn(0x7f90d00fc300 msgr2=0x7f90d00fe740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 -- 172.21.15.90:0/265254317 shutdown_connections 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.284+0000 7f90d493a700 1 -- 172.21.15.90:0/265254317 wait complete. 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 Processor -- start 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 -- start start 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 0x7f90d0196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 0x7f90d019e320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d019e8f0 con 0x7f90d0101f80 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d019ea30 con 0x7f90d0102920 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90d493a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d019ebc0 con 0x7f90d0100d90 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54542/0 (socket says 172.21.15.90:54542) 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 -- 172.21.15.90:0/2323435512 learned_addr learned my addr 172.21.15.90:0/2323435512 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:02.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90ced9d700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 0x7f90d019e320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90ce59c700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 0x7f90d0196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 -- 172.21.15.90:0/2323435512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 msgr2=0x7f90d0196dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 0x7f90d0196dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 -- 172.21.15.90:0/2323435512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 msgr2=0x7f90d019e320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 0x7f90d019e320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90cdd9b700 1 -- 172.21.15.90:0/2323435512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90c8009d00 con 0x7f90d0101f80 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90ced9d700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 0x7f90d019e320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.285+0000 7f90ce59c700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 0x7f90d0196dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90cdd9b700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f90c000c920 tx=0x7f90c000cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90c0007b00 con 0x7f90d0101f80 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f90c0004cf0 con 0x7f90d0101f80 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90d019eea0 con 0x7f90d0101f80 2024-04-06T14:04:02.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90c0005520 con 0x7f90d0101f80 2024-04-06T14:04:02.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.286+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90d019f320 con 0x7f90d0101f80 2024-04-06T14:04:02.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.287+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90d0108140 con 0x7f90d0101f80 2024-04-06T14:04:02.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.288+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f90c0004750 con 0x7f90d0101f80 2024-04-06T14:04:02.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.288+0000 7f90c77fe700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 0x7f90bc0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:02.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.288+0000 7f90ce59c700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 0x7f90bc0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:02.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.288+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f90c00cc4d0 con 0x7f90d0101f80 2024-04-06T14:04:02.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.289+0000 7f90ce59c700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 0x7f90bc0a35e0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f90d0101de0 tx=0x7f90b8005f90 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:02.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.293+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f90c0095d10 con 0x7f90d0101f80 2024-04-06T14:04:02.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-06T14:04:02.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-06T14:04:02.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: osdmap e79: 12 total, 12 up, 12 in 2024-04-06T14:04:02.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: osdmap e80: 12 total, 12 up, 12 in 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-06T14:04:02.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/487441945' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:02.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/131368936' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: osdmap e79: 12 total, 12 up, 12 in 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: osdmap e80: 12 total, 12 up, 12 in 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3785683434' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-04-06T14:04:02.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/487441945' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:02.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:02.541+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7f90d0067b00 con 0x7f90d0101f80 2024-04-06T14:04:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1205501394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: pgmap v219: 97 pgs: 7 creating+peering, 20 active+clean, 70 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 0 op/s 2024-04-06T14:04:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: osdmap e81: 12 total, 12 up, 12 in 2024-04-06T14:04:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:active 2024-04-06T14:04:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:03 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1205501394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: pgmap v219: 97 pgs: 7 creating+peering, 20 active+clean, 70 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 0 op/s 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: osdmap e81: 12 total, 12 up, 12 in 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:active 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:03 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-06T14:04:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.491+0000 7f90c77fe700 1 -- 172.21.15.90:0/2323435512 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v17) v1 ==== 118+0+0 (secure 0 0 0) 0x7f90c0099330 con 0x7f90d0101f80 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 msgr2=0x7f90bc0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 0x7f90bc0a35e0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f90d0101de0 tx=0x7f90b8005f90 comp rx=0 tx=0).stop 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 msgr2=0x7f90d0197300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f90c000c920 tx=0x7f90c000cce0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 shutdown_connections 2024-04-06T14:04:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90bc0a1120 0x7f90bc0a35e0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d0102920 0x7f90d019e320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d0100d90 0x7f90d0196dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 --2- 172.21.15.90:0/2323435512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d0101f80 0x7f90d0197300 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 >> 172.21.15.90:0/2323435512 conn(0x7f90d00fc300 msgr2=0x7f90d0105280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 shutdown_connections 2024-04-06T14:04:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.494+0000 7f90d493a700 1 -- 172.21.15.90:0/2323435512 wait complete. 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1205501394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: pgmap v219: 97 pgs: 7 creating+peering, 20 active+clean, 70 unknown; 577 KiB data, 346 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 0 op/s 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: osdmap e81: 12 total, 12 up, 12 in 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] up:active 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:03 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-04-06T14:04:03.509 DEBUG:tasks.cephfs.filesystem:fs_config: {'max_mds': 3, 'session_timeout': 300, 'standby_replay': True, 'subvols': {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'}} 2024-04-06T14:04:03.509 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 3 2024-04-06T14:04:03.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 -- 172.21.15.90:0/2041813664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 msgr2=0x7f0f201011a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 --2- 172.21.15.90:0/2041813664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f201011a0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f0f1c0099c0 tx=0x7f0f1c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 -- 172.21.15.90:0/2041813664 shutdown_connections 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 --2- 172.21.15.90:0/2041813664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20102910 0x7f0f201091c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 --2- 172.21.15.90:0/2041813664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f201011a0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 --2- 172.21.15.90:0/2041813664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20101f70 0x7f0f201023d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 -- 172.21.15.90:0/2041813664 >> 172.21.15.90:0/2041813664 conn(0x7f0f200fc2f0 msgr2=0x7f0f200fe730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 -- 172.21.15.90:0/2041813664 shutdown_connections 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.639+0000 7f0f27e82700 1 -- 172.21.15.90:0/2041813664 wait complete. 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 Processor -- start 2024-04-06T14:04:03.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 -- start start 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20101f70 0x7f0f20197240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20102910 0x7f0f2019e250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f2019e820 con 0x7f0f20100d80 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f2019e990 con 0x7f0f20102910 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.640+0000 7f0f27e82700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f2019eb50 con 0x7f0f20101f70 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34000/0 (socket says 172.21.15.90:34000) 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 -- 172.21.15.90:0/4113095590 learned_addr learned my addr 172.21.15.90:0/4113095590 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f2641f700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20102910 0x7f0f2019e250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f2541d700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20101f70 0x7f0f20197240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 -- 172.21.15.90:0/4113095590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20101f70 msgr2=0x7f0f20197240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20101f70 0x7f0f20197240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 -- 172.21.15.90:0/4113095590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20102910 msgr2=0x7f0f2019e250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20102910 0x7f0f2019e250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 -- 172.21.15.90:0/4113095590 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f1c0095f0 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f25c1e700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f0f1c003820 tx=0x7f0f1c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f1c01d070 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0f1c0054f0 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0f2019edd0 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f1c004a00 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.641+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0f2019f1d0 con 0x7f0f20100d80 2024-04-06T14:04:03.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.642+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f201081a0 con 0x7f0f20100d80 2024-04-06T14:04:03.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.643+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f0f1c003b50 con 0x7f0f20100d80 2024-04-06T14:04:03.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.644+0000 7f0f16ffd700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 0x7f0f0c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:03.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.644+0000 7f0f2541d700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 0x7f0f0c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:03.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.644+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f0f1c0cd690 con 0x7f0f20100d80 2024-04-06T14:04:03.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.644+0000 7f0f2541d700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 0x7f0f0c0a35e0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f0f10005950 tx=0x7f0f100058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:03.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.648+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0f1c096f60 con 0x7f0f20100d80 2024-04-06T14:04:03.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:03.893+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"} v 0) v1 -- 0x7f0f2002cd70 con 0x7f0f20100d80 2024-04-06T14:04:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:04 smithi090 ceph-mon[43420]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-06T14:04:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:04 smithi090 ceph-mon[43420]: Cluster is now healthy 2024-04-06T14:04:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:04 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-06T14:04:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:04 smithi090 ceph-mon[43420]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:04 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-06T14:04:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:04 smithi134 ceph-mon[47549]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-06T14:04:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:04 smithi134 ceph-mon[47549]: Cluster is now healthy 2024-04-06T14:04:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:04 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-06T14:04:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:04 smithi134 ceph-mon[47549]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:04 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-06T14:04:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.495+0000 7f0f16ffd700 1 -- 172.21.15.90:0/4113095590 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]=0 v18) v1 ==== 105+0+0 (secure 0 0 0) 0x7f0f1c09a580 con 0x7f0f20100d80 2024-04-06T14:04:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:04 smithi105 ceph-mon[46969]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-06T14:04:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:04 smithi105 ceph-mon[46969]: Cluster is now healthy 2024-04-06T14:04:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:04 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2323435512' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-04-06T14:04:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:04 smithi105 ceph-mon[46969]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:04 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]: dispatch 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 msgr2=0x7f0f0c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 0x7f0f0c0a35e0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f0f10005950 tx=0x7f0f100058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 msgr2=0x7f0f20196d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f0f1c003820 tx=0x7f0f1c004730 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 shutdown_connections 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0f0c0a1120 0x7f0f0c0a35e0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f20102910 0x7f0f2019e250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0f20100d80 0x7f0f20196d00 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 --2- 172.21.15.90:0/4113095590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0f20101f70 0x7f0f20197240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 >> 172.21.15.90:0/4113095590 conn(0x7f0f200fc2f0 msgr2=0x7f0f201052e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:04.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 shutdown_connections 2024-04-06T14:04:04.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.498+0000 7f0f27e82700 1 -- 172.21.15.90:0/4113095590 wait complete. 2024-04-06T14:04:04.515 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay true 2024-04-06T14:04:04.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 -- 172.21.15.90:0/175091540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 msgr2=0x7f51801080b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:04.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 --2- 172.21.15.90:0/175091540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f51801080b0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f517800b0d0 tx=0x7f517800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 -- 172.21.15.90:0/175091540 shutdown_connections 2024-04-06T14:04:04.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 --2- 172.21.15.90:0/175091540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f51801080b0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 --2- 172.21.15.90:0/175091540 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 0x7f5180100090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 --2- 172.21.15.90:0/175091540 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 0x7f51801012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 -- 172.21.15.90:0/175091540 >> 172.21.15.90:0/175091540 conn(0x7f51800fb200 msgr2=0x7f51800fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 -- 172.21.15.90:0/175091540 shutdown_connections 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.646+0000 7f5184f3e700 1 -- 172.21.15.90:0/175091540 wait complete. 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 Processor -- start 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 -- start start 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 0x7f5180068940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 0x7f518019bbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f518019e9e0 con 0x7f5180101800 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f518019eb50 con 0x7f5180100e60 2024-04-06T14:04:04.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f5184f3e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f518019ed10 con 0x7f51800ffc70 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34010/0 (socket says 172.21.15.90:34010) 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 -- 172.21.15.90:0/1572752497 learned_addr learned my addr 172.21.15.90:0/1572752497 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517dd9b700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 0x7f518019bbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517e59c700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 0x7f5180068940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 -- 172.21.15.90:0/1572752497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 msgr2=0x7f5180068940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 0x7f5180068940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 -- 172.21.15.90:0/1572752497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 msgr2=0x7f518019bbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 0x7f518019bbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 -- 172.21.15.90:0/1572752497 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5178009d00 con 0x7f5180101800 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.647+0000 7f517ed9d700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f5178007bd0 tx=0x7f5178009440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f517800e040 con 0x7f5180101800 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f517801dd50 con 0x7f5180101800 2024-04-06T14:04:04.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f518019ef30 con 0x7f5180101800 2024-04-06T14:04:04.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51780123f0 con 0x7f5180101800 2024-04-06T14:04:04.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f518019f350 con 0x7f5180101800 2024-04-06T14:04:04.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.648+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f51801070a0 con 0x7f5180101800 2024-04-06T14:04:04.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.650+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f517801b550 con 0x7f5180101800 2024-04-06T14:04:04.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.650+0000 7f516f7fe700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 0x7f51680a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:04.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.650+0000 7f517e59c700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 0x7f51680a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:04.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.650+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f51780ced50 con 0x7f5180101800 2024-04-06T14:04:04.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.651+0000 7f517e59c700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 0x7f51680a35e0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f5180100cc0 tx=0x7f5174009190 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:04.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.654+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f51780986c0 con 0x7f5180101800 2024-04-06T14:04:04.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:04.899+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) v1 -- 0x7f5180067b00 con 0x7f5180101800 2024-04-06T14:04:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: pgmap v221: 97 pgs: 7 creating+peering, 76 active+clean, 14 unknown; 578 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s wr, 8 op/s 2024-04-06T14:04:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-06T14:04:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:05 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: pgmap v221: 97 pgs: 7 creating+peering, 76 active+clean, 14 unknown; 578 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s wr, 8 op/s 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-06T14:04:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:05 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-06T14:04:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: pgmap v221: 97 pgs: 7 creating+peering, 76 active+clean, 14 unknown; 578 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s wr, 8 op/s 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4113095590' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "3"}]': finished 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: daemon mds.i assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: fsmap cephfs:2 {0=b=up:active,1=i=up:creating} 10 up:standby 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: daemon mds.i is now active in filesystem cephfs as rank 1 2024-04-06T14:04:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-06T14:04:05.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.500+0000 7f516f7fe700 1 -- 172.21.15.90:0/1572752497 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v20) v1 ==== 121+0+0 (secure 0 0 0) 0x7f517809bce0 con 0x7f5180101800 2024-04-06T14:04:05.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 msgr2=0x7f51680a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:05.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 0x7f51680a35e0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f5180100cc0 tx=0x7f5174009190 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 msgr2=0x7f518019e1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f5178007bd0 tx=0x7f5178009440 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 shutdown_connections 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51680a1120 0x7f51680a35e0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5180101800 0x7f518019e1f0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51800ffc70 0x7f5180068940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 --2- 172.21.15.90:0/1572752497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5180100e60 0x7f518019bbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 >> 172.21.15.90:0/1572752497 conn(0x7f51800fb200 msgr2=0x7f51801041e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 shutdown_connections 2024-04-06T14:04:05.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.504+0000 7f5184f3e700 1 -- 172.21.15.90:0/1572752497 wait complete. 2024-04-06T14:04:05.521 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs session_timeout 300 2024-04-06T14:04:05.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 -- 172.21.15.90:0/3412916864 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 msgr2=0x7fde881080d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:05.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 --2- 172.21.15.90:0/3412916864 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde881080d0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fde8000b0d0 tx=0x7fde8000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 -- 172.21.15.90:0/3412916864 shutdown_connections 2024-04-06T14:04:05.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 --2- 172.21.15.90:0/3412916864 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde881080d0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 --2- 172.21.15.90:0/3412916864 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 0x7fde881000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 --2- 172.21.15.90:0/3412916864 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 0x7fde881012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 -- 172.21.15.90:0/3412916864 >> 172.21.15.90:0/3412916864 conn(0x7fde880fb200 msgr2=0x7fde880fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 -- 172.21.15.90:0/3412916864 shutdown_connections 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 -- 172.21.15.90:0/3412916864 wait complete. 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.652+0000 7fde905e1700 1 Processor -- start 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 -- start start 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 0x7fde88069da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 0x7fde8819bae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde8819e8f0 con 0x7fde88101820 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde8819ea60 con 0x7fde88100e80 2024-04-06T14:04:05.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde905e1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde8819ec20 con 0x7fde880ffc90 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34018/0 (socket says 172.21.15.90:34018) 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 -- 172.21.15.90:0/2489310421 learned_addr learned my addr 172.21.15.90:0/2489310421 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8db7c700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 0x7fde8819bae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8e37d700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 0x7fde88069da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 -- 172.21.15.90:0/2489310421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 msgr2=0x7fde88069da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 0x7fde88069da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 -- 172.21.15.90:0/2489310421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 msgr2=0x7fde8819bae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 0x7fde8819bae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 -- 172.21.15.90:0/2489310421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde80009d00 con 0x7fde88101820 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.653+0000 7fde8eb7e700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fde80007b60 tx=0x7fde80008f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde8000e040 con 0x7fde88101820 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fde8000be30 con 0x7fde88101820 2024-04-06T14:04:05.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde8819ee40 con 0x7fde88101820 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde8001bae0 con 0x7fde88101820 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde8819f210 con 0x7fde88101820 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.654+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde88107040 con 0x7fde88101820 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.656+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7fde8001d670 con 0x7fde88101820 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.656+0000 7fde7f7fe700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 0x7fde740a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:05.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.657+0000 7fde8e37d700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 0x7fde740a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:05.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.657+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fde800ce770 con 0x7fde88101820 2024-04-06T14:04:05.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.657+0000 7fde8e37d700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 0x7fde740a35e0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fde88100ce0 tx=0x7fde84009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:05.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.660+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fde800980e0 con 0x7fde88101820 2024-04-06T14:04:05.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:05.902+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"} v 0) v1 -- 0x7fde8810b4b0 con 0x7fde88101820 2024-04-06T14:04:06.247 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: 2024-04-06T14:04:05.988+0000 7f2bd6847700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-04-06T14:04:06.247 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:04:05 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: 2024-04-06T14:04:05.988+0000 7f2bd6847700 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2024-04-06T14:04:06.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.509+0000 7fde7f7fe700 1 -- 172.21.15.90:0/2489310421 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]=0 v22) v1 ==== 115+0+0 (secure 0 0 0) 0x7fde8009b700 con 0x7fde88101820 2024-04-06T14:04:06.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 msgr2=0x7fde740a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 0x7fde740a35e0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fde88100ce0 tx=0x7fde84009260 comp rx=0 tx=0).stop 2024-04-06T14:04:06.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 msgr2=0x7fde8819e100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fde80007b60 tx=0x7fde80008f10 comp rx=0 tx=0).stop 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 shutdown_connections 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde740a1120 0x7fde740a35e0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde88101820 0x7fde8819e100 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde880ffc90 0x7fde88069da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 --2- 172.21.15.90:0/2489310421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde88100e80 0x7fde8819bae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.511+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 >> 172.21.15.90:0/2489310421 conn(0x7fde880fb200 msgr2=0x7fde88104180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.512+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 shutdown_connections 2024-04-06T14:04:06.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.512+0000 7fde905e1700 1 -- 172.21.15.90:0/2489310421 wait complete. 2024-04-06T14:04:06.525 DEBUG:tasks.cephfs.filesystem:Creating {'create': 2, 'subvol_options': '--namespace-isolated --size 25000000000'} subvols for filesystem 'cephfs' 2024-04-06T14:04:06.526 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolumegroup create cephfs qa 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 -- 172.21.15.90:0/3026840965 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 msgr2=0x7f9444108ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 --2- 172.21.15.90:0/3026840965 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f9444108ee0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f943c00b0a0 tx=0x7f943c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 -- 172.21.15.90:0/3026840965 shutdown_connections 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 --2- 172.21.15.90:0/3026840965 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f9444108ee0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 --2- 172.21.15.90:0/3026840965 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 0x7f9444103c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 --2- 172.21.15.90:0/3026840965 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 0x7f9444106530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 -- 172.21.15.90:0/3026840965 >> 172.21.15.90:0/3026840965 conn(0x7f94440fb200 msgr2=0x7f94440fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 -- 172.21.15.90:0/3026840965 shutdown_connections 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.655+0000 7f944a4a2700 1 -- 172.21.15.90:0/3026840965 wait complete. 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 Processor -- start 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 -- start start 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 0x7f9444196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 0x7f9444197300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f944419e8f0 con 0x7f9444106a70 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f944419ea30 con 0x7f9444101810 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f944a4a2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f944419ebc0 con 0x7f9444104140 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:06.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34026/0 (socket says 172.21.15.90:34026) 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 -- 172.21.15.90:0/125237731 learned_addr learned my addr 172.21.15.90:0/125237731 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9443fff700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 0x7f9444196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f94437fe700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 0x7f9444197300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 -- 172.21.15.90:0/125237731 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 msgr2=0x7f9444197300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 0x7f9444197300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 -- 172.21.15.90:0/125237731 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 msgr2=0x7f9444196dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 0x7f9444196dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 -- 172.21.15.90:0/125237731 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f943c009d00 con 0x7f9444106a70 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.656+0000 7f9448a3f700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f943c009030 tx=0x7f943c00bab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.657+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f943c00e040 con 0x7f9444106a70 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.657+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f943c004c50 con 0x7f9444106a70 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.657+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f944419ee40 con 0x7f9444106a70 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.657+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f943c0052a0 con 0x7f9444106a70 2024-04-06T14:04:06.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.657+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f944419f210 con 0x7f9444106a70 2024-04-06T14:04:06.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.658+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f94440fcd80 con 0x7f9444106a70 2024-04-06T14:04:06.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.659+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f943c005400 con 0x7f9444106a70 2024-04-06T14:04:06.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.659+0000 7f94417fa700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 0x7f942c0a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:06.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.659+0000 7f9443fff700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 0x7f942c0a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:06.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.659+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f943c0ce280 con 0x7f9444106a70 2024-04-06T14:04:06.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.660+0000 7f9443fff700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 0x7f942c0a35e0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f9438009c60 tx=0x7f9438009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:06.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.663+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f943c097bf0 con 0x7f9444106a70 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:active 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: Cluster is now healthy 2024-04-06T14:04:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-06T14:04:06.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-06T14:04:06.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-06T14:04:06.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-06T14:04:06.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-mon[43420]: pgmap v222: 97 pgs: 97 active+clean; 579 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s wr, 9 op/s 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:active 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: Cluster is now healthy 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-06T14:04:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:06 smithi134 ceph-mon[47549]: pgmap v222: 97 pgs: 97 active+clean; 579 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s wr, 9 op/s 2024-04-06T14:04:06.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.895+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}) v1 -- 0x7f94440611f0 con 0x7f942c0a1120 2024-04-06T14:04:06.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.919+0000 7f94417fa700 1 -- 172.21.15.90:0/125237731 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f94440611f0 con 0x7f942c0a1120 2024-04-06T14:04:06.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 msgr2=0x7f942c0a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 0x7f942c0a35e0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f9438009c60 tx=0x7f9438009260 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 msgr2=0x7f944419e320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f943c009030 tx=0x7f943c00bab0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 shutdown_connections 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f942c0a1120 0x7f942c0a35e0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9444106a70 0x7f944419e320 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9444101810 0x7f9444196dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 --2- 172.21.15.90:0/125237731 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9444104140 0x7f9444197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 >> 172.21.15.90:0/125237731 conn(0x7f94440fb200 msgr2=0x7f9444102150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 shutdown_connections 2024-04-06T14:04:06.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:06.922+0000 7f944a4a2700 1 -- 172.21.15.90:0/125237731 wait complete. 2024-04-06T14:04:06.933 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_0 --group_name qa 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] up:active 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1572752497' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: daemon mds.h assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: Cluster is now healthy 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: fsmap cephfs:2 {0=b=up:active,1=i=up:active} 10 up:standby 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:creating} 9 up:standby 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: daemon mds.h is now active in filesystem cephfs as rank 2 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2024-04-06T14:04:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:06 smithi105 ceph-mon[46969]: pgmap v222: 97 pgs: 97 active+clean; 579 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.3 KiB/s wr, 9 op/s 2024-04-06T14:04:07.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 -- 172.21.15.90:0/4166372709 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c101820 msgr2=0x7f291c1080d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 --2- 172.21.15.90:0/4166372709 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c101820 0x7f291c1080d0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f291400b0d0 tx=0x7f291400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 -- 172.21.15.90:0/4166372709 shutdown_connections 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 --2- 172.21.15.90:0/4166372709 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c101820 0x7f291c1080d0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 --2- 172.21.15.90:0/4166372709 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c0ffc90 0x7f291c1000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 --2- 172.21.15.90:0/4166372709 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c100e80 0x7f291c1012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 -- 172.21.15.90:0/4166372709 >> 172.21.15.90:0/4166372709 conn(0x7f291c0fb200 msgr2=0x7f291c0fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 -- 172.21.15.90:0/4166372709 shutdown_connections 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.062+0000 7f29240d4700 1 -- 172.21.15.90:0/4166372709 wait complete. 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 Processor -- start 2024-04-06T14:04:07.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 -- start start 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c100e80 0x7f291c19bb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 0x7f291c19e130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f291c19e8d0 con 0x7f291c0ffc90 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f291c19ea40 con 0x7f291c100e80 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f29240d4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f291c19ec00 con 0x7f291c101820 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34072/0 (socket says 172.21.15.90:34072) 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 -- 172.21.15.90:0/1840135513 learned_addr learned my addr 172.21.15.90:0/1840135513 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2922671700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 0x7f291c19e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 -- 172.21.15.90:0/1840135513 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 msgr2=0x7f291c19e130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 0x7f291c19e130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 -- 172.21.15.90:0/1840135513 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c100e80 msgr2=0x7f291c19bb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f292166f700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c100e80 0x7f291c19bb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c100e80 0x7f291c19bb40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 -- 172.21.15.90:0/1840135513 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2914009d00 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2922671700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 0x7f291c19e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f2921e70700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f291000d980 tx=0x7f291000dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.063+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f291000de60 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.064+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f29100044e0 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.064+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f291c19eee0 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.064+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2910003be0 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.064+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f291c19f360 con 0x7f291c0ffc90 2024-04-06T14:04:07.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.064+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f291c107040 con 0x7f291c0ffc90 2024-04-06T14:04:07.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.066+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f2910009bb0 con 0x7f291c0ffc90 2024-04-06T14:04:07.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.066+0000 7f290effd700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 0x7f29080a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.066+0000 7f292166f700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 0x7f29080a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.066+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f29100cc820 con 0x7f291c0ffc90 2024-04-06T14:04:07.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.067+0000 7f292166f700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 0x7f29080a3660 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f2918005950 tx=0x7f29180058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.069+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f2910096180 con 0x7f291c0ffc90 2024-04-06T14:04:07.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:04:06.901+0000 7f6a24f1a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:04:07.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:04:06.901+0000 7f6a24f1a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:04:07.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:04:06.901+0000 7f6a24f1a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:04:07.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:04:06.901+0000 7f6a24f1a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:04:07.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:04:06 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:04:06.901+0000 7f6a24f1a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:04:07.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.300+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}) v1 -- 0x7f291c02cd50 con 0x7f29080a11a0 2024-04-06T14:04:07.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.322+0000 7f290effd700 1 -- 172.21.15.90:0/1840135513 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f291c02cd50 con 0x7f29080a11a0 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 msgr2=0x7f29080a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 0x7f29080a3660 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f2918005950 tx=0x7f29180058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 msgr2=0x7f291c069b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f291000d980 tx=0x7f291000dd40 comp rx=0 tx=0).stop 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 shutdown_connections 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f29080a11a0 0x7f29080a3660 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f291c101820 0x7f291c19e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f291c0ffc90 0x7f291c069b20 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 --2- 172.21.15.90:0/1840135513 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f291c100e80 0x7f291c19bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 >> 172.21.15.90:0/1840135513 conn(0x7f291c0fb200 msgr2=0x7f291c104180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:07.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 shutdown_connections 2024-04-06T14:04:07.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.324+0000 7f29240d4700 1 -- 172.21.15.90:0/1840135513 wait complete. 2024-04-06T14:04:07.336 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_0 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- 172.21.15.90:0/61147012 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd8101820 msgr2=0x7f6fd81080d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 --2- 172.21.15.90:0/61147012 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd8101820 0x7f6fd81080d0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f6fd000b0d0 tx=0x7f6fd000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- 172.21.15.90:0/61147012 shutdown_connections 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 --2- 172.21.15.90:0/61147012 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd8101820 0x7f6fd81080d0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 --2- 172.21.15.90:0/61147012 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd80ffc90 0x7f6fd81000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 --2- 172.21.15.90:0/61147012 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8100e80 0x7f6fd81012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- 172.21.15.90:0/61147012 >> 172.21.15.90:0/61147012 conn(0x7f6fd80fb200 msgr2=0x7f6fd80fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- 172.21.15.90:0/61147012 shutdown_connections 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- 172.21.15.90:0/61147012 wait complete. 2024-04-06T14:04:07.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 Processor -- start 2024-04-06T14:04:07.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.464+0000 7f6fdd331700 1 -- start start 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 0x7f6fd819bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8101820 0x7f6fd819e150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819e940 con 0x7f6fd80ffc90 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819eab0 con 0x7f6fd8101820 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819ec70 con 0x7f6fd8100e80 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34088/0 (socket says 172.21.15.90:34088) 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 -- 172.21.15.90:0/4003647454 learned_addr learned my addr 172.21.15.90:0/4003647454 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd67fc700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 0x7f6fd819bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd77fe700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8101820 0x7f6fd819e150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 -- 172.21.15.90:0/4003647454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 msgr2=0x7f6fd819bb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 0x7f6fd819bb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 -- 172.21.15.90:0/4003647454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8101820 msgr2=0x7f6fd819e150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8101820 0x7f6fd819e150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 -- 172.21.15.90:0/4003647454 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fd0009d00 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd67fc700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 0x7f6fd819bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fd6ffd700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f6fcc00d980 tx=0x7f6fcc00dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fcc00bb60 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6fcc007620 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fd819eef0 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fcc00f620 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.465+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fd819f310 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.466+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fd8107040 con 0x7f6fd80ffc90 2024-04-06T14:04:07.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.467+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f6fcc010460 con 0x7f6fd80ffc90 2024-04-06T14:04:07.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.468+0000 7f6fbffff700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 0x7f6fc00a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.468+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f6fcc0cd060 con 0x7f6fd80ffc90 2024-04-06T14:04:07.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.468+0000 7f6fd67fc700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 0x7f6fc00a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.468+0000 7f6fd67fc700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 0x7f6fc00a35e0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f6fd806a340 tx=0x7f6fc8009550 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.471+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6fcc096940 con 0x7f6fd80ffc90 2024-04-06T14:04:07.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.701+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}) v1 -- 0x7f6fd802cd50 con 0x7f6fc00a1120 2024-04-06T14:04:07.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.705+0000 7f6fbffff700 1 -- 172.21.15.90:0/4003647454 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+54 (secure 0 0 0) 0x7f6fd802cd50 con 0x7f6fc00a1120 2024-04-06T14:04:07.705 INFO:teuthology.orchestra.run.smithi090.stdout:/volumes/qa/sv_0/12e813d6-fb52-48a9-96ad-b31f48191076 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 msgr2=0x7f6fc00a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 0x7f6fc00a35e0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f6fd806a340 tx=0x7f6fc8009550 comp rx=0 tx=0).stop 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 msgr2=0x7f6fd8069da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f6fcc00d980 tx=0x7f6fcc00dd40 comp rx=0 tx=0).stop 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 shutdown_connections 2024-04-06T14:04:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6fc00a1120 0x7f6fc00a35e0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd8101820 0x7f6fd819e150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd80ffc90 0x7f6fd8069da0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 --2- 172.21.15.90:0/4003647454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd8100e80 0x7f6fd819bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.707+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 >> 172.21.15.90:0/4003647454 conn(0x7f6fd80fb200 msgr2=0x7f6fd8104180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.708+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 shutdown_connections 2024-04-06T14:04:07.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.708+0000 7f6fdd331700 1 -- 172.21.15.90:0/4003647454 wait complete. 2024-04-06T14:04:07.718 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:active 2024-04-06T14:04:07.718 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-06T14:04:07.719 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-06T14:04:07.719 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:04:07.719 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:07.719 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:07 smithi090 ceph-mon[43420]: from='client.14751 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:07.720 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume create cephfs sv_1 --group_name qa 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 -- 172.21.15.90:0/312078722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c1041c0 msgr2=0x7f988c1065b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 --2- 172.21.15.90:0/312078722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c1041c0 0x7f988c1065b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f9880009990 tx=0x7f9880009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 -- 172.21.15.90:0/312078722 shutdown_connections 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 --2- 172.21.15.90:0/312078722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c106af0 0x7f988c108ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 --2- 172.21.15.90:0/312078722 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c101890 0x7f988c103c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 --2- 172.21.15.90:0/312078722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c1041c0 0x7f988c1065b0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 -- 172.21.15.90:0/312078722 >> 172.21.15.90:0/312078722 conn(0x7f988c0fb200 msgr2=0x7f988c0fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 -- 172.21.15.90:0/312078722 shutdown_connections 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.847+0000 7f9893f2c700 1 -- 172.21.15.90:0/312078722 wait complete. 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 Processor -- start 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 -- start start 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 0x7f988c196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c1041c0 0x7f988c1972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988c19e8d0 con 0x7f988c106af0 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988c19ea10 con 0x7f988c101890 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9893f2c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988c19eba0 con 0x7f988c1041c0 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34102/0 (socket says 172.21.15.90:34102) 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 -- 172.21.15.90:0/3765381244 learned_addr learned my addr 172.21.15.90:0/3765381244 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 -- 172.21.15.90:0/3765381244 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c1041c0 msgr2=0x7f988c1972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98914c7700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c1041c0 0x7f988c1972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9891cc8700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 0x7f988c196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c1041c0 0x7f988c1972e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 -- 172.21.15.90:0/3765381244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 msgr2=0x7f988c196da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 0x7f988c196da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 -- 172.21.15.90:0/3765381244 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98800095f0 con 0x7f988c106af0 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f9891cc8700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 0x7f988c196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f98924c9700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f988400ada0 tx=0x7f988400c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f988400f070 con 0x7f988c106af0 2024-04-06T14:04:07.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.848+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9884007b30 con 0x7f988c106af0 2024-04-06T14:04:07.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.849+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f988c19ee80 con 0x7f988c106af0 2024-04-06T14:04:07.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.849+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9884005440 con 0x7f988c106af0 2024-04-06T14:04:07.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.849+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f988c19f300 con 0x7f988c106af0 2024-04-06T14:04:07.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.850+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f988c0fcd80 con 0x7f988c106af0 2024-04-06T14:04:07.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.850+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f9884004ad0 con 0x7f988c106af0 2024-04-06T14:04:07.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.851+0000 7f987effd700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 0x7f98780a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:07.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.851+0000 7f9891cc8700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 0x7f98780a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:07.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.851+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f98840cbe20 con 0x7f988c106af0 2024-04-06T14:04:07.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.851+0000 7f9891cc8700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 0x7f98780a35e0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f9888007900 tx=0x7f9888008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:07.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:07.855+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9884095790 con 0x7f988c106af0 2024-04-06T14:04:07.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:active 2024-04-06T14:04:07.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-06T14:04:07.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-06T14:04:07.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:04:07.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:07.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:07 smithi134 ceph-mon[47549]: from='client.14751 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:07.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] up:active 2024-04-06T14:04:07.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2489310421' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2024-04-06T14:04:07.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 9 up:standby 2024-04-06T14:04:07.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:04:07.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: from='client.14739 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup create", "vol_name": "cephfs", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:07.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:07 smithi105 ceph-mon[46969]: from='client.14751 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_0", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.085+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}) v1 -- 0x7f988c02cd20 con 0x7f98780a1120 2024-04-06T14:04:08.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.108+0000 7f987effd700 1 -- 172.21.15.90:0/3765381244 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f988c02cd20 con 0x7f98780a1120 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 msgr2=0x7f98780a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 0x7f98780a35e0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f9888007900 tx=0x7f9888008040 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 msgr2=0x7f988c19e300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f988400ada0 tx=0x7f988400c330 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 shutdown_connections 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f98780a1120 0x7f98780a35e0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f988c106af0 0x7f988c19e300 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f988c101890 0x7f988c196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 --2- 172.21.15.90:0/3765381244 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f988c1041c0 0x7f988c1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 >> 172.21.15.90:0/3765381244 conn(0x7f988c0fb200 msgr2=0x7f988c103720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:08.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 shutdown_connections 2024-04-06T14:04:08.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.111+0000 7f9893f2c700 1 -- 172.21.15.90:0/3765381244 wait complete. 2024-04-06T14:04:08.123 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs subvolume getpath cephfs --group_name qa sv_1 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 -- 172.21.15.90:0/1287973206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 msgr2=0x7f2be01000b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1287973206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be01000b0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f2bd40099c0 tx=0x7f2bd4009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 -- 172.21.15.90:0/1287973206 shutdown_connections 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1287973206 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 0x7f2be01080d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1287973206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be01000b0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1287973206 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 0x7f2be01012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 -- 172.21.15.90:0/1287973206 >> 172.21.15.90:0/1287973206 conn(0x7f2be00fb200 msgr2=0x7f2be00fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 -- 172.21.15.90:0/1287973206 shutdown_connections 2024-04-06T14:04:08.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 -- 172.21.15.90:0/1287973206 wait complete. 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.251+0000 7f2be4af9700 1 Processor -- start 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- start start 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 0x7f2be0102260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 0x7f2be01027a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2be0102f40 con 0x7f2be00ffc90 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2be01030b0 con 0x7f2be0101820 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2be01a0e30 con 0x7f2be0100e80 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34108/0 (socket says 172.21.15.90:34108) 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 -- 172.21.15.90:0/1060518395 learned_addr learned my addr 172.21.15.90:0/1060518395 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bddd9b700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 0x7f2be0102260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bded9d700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 0x7f2be01027a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 -- 172.21.15.90:0/1060518395 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 msgr2=0x7f2be0102260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 0x7f2be0102260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 -- 172.21.15.90:0/1060518395 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 msgr2=0x7f2be01027a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 0x7f2be01027a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 -- 172.21.15.90:0/1060518395 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2bd40095f0 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bde59c700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f2bd4006010 tx=0x7f2bd4004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bd401d070 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2bd4003f90 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2be01a1060 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bd4022d80 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.252+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2be01a1420 con 0x7f2be00ffc90 2024-04-06T14:04:08.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.253+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2be0067270 con 0x7f2be00ffc90 2024-04-06T14:04:08.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.254+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 16) v1 ==== 135873+0+0 (secure 0 0 0) 0x7f2bd4022470 con 0x7f2be00ffc90 2024-04-06T14:04:08.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.255+0000 7f2bcf7fe700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 0x7f2bc80a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.255+0000 7f2bddd9b700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 0x7f2bc80a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.255+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f2bd40ce1c0 con 0x7f2be00ffc90 2024-04-06T14:04:08.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.255+0000 7f2bddd9b700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 0x7f2bc80a35e0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f2bd0005950 tx=0x7f2bd000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:08.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.259+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f2bd4097b30 con 0x7f2be00ffc90 2024-04-06T14:04:08.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.487+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f2be0069740 con 0x7f2bc80a1120 2024-04-06T14:04:08.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.491+0000 7f2bcf7fe700 1 -- 172.21.15.90:0/1060518395 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+54 (secure 0 0 0) 0x7f2be0069740 con 0x7f2bc80a1120 2024-04-06T14:04:08.491 INFO:teuthology.orchestra.run.smithi090.stdout:/volumes/qa/sv_1/b6ecb66e-1758-4685-8e5c-896930794149 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 msgr2=0x7f2bc80a35e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 0x7f2bc80a35e0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f2bd0005950 tx=0x7f2bd000b410 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 msgr2=0x7f2be0103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f2bd4006010 tx=0x7f2bd4004730 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 shutdown_connections 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f2bc80a1120 0x7f2bc80a35e0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2be0101820 0x7f2be01027a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2be00ffc90 0x7f2be0103bd0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 --2- 172.21.15.90:0/1060518395 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2be0100e80 0x7f2be0102260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 >> 172.21.15.90:0/1060518395 conn(0x7f2be00fb200 msgr2=0x7f2be0069030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:08.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 shutdown_connections 2024-04-06T14:04:08.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.493+0000 7f2be4af9700 1 -- 172.21.15.90:0/1060518395 wait complete. 2024-04-06T14:04:08.505 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:04:08.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2200887826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc0ffe10 msgr2=0x7f37cc100290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2200887826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc0ffe10 0x7f37cc100290 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f37c8009990 tx=0x7f37c8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2200887826 shutdown_connections 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2200887826 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc1007d0 0x7f37cc108070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2200887826 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc1015e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2200887826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc0ffe10 0x7f37cc100290 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2200887826 >> 172.21.15.90:0/2200887826 conn(0x7f37cc0fa8c0 msgr2=0x7f37cc0fcce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.631+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2200887826 shutdown_connections 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.632+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2200887826 wait complete. 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.632+0000 7f37d3ecd700 1 Processor -- start 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.632+0000 7f37d3ecd700 1 -- start start 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.632+0000 7f37d3ecd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 0x7f37cc0ff880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.632+0000 7f37d3ecd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc19df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc19e6e0 con 0x7f37cc1007d0 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc19e850 con 0x7f37cc0ffe10 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37cc19ea10 con 0x7f37cc1011c0 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34130/0 (socket says 172.21.15.90:34130) 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 -- 172.21.15.90:0/2890994672 learned_addr learned my addr 172.21.15.90:0/2890994672 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1c69700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 0x7f37cc0ff880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d246a700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc19df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 -- 172.21.15.90:0/2890994672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 msgr2=0x7f37cc19df40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc19df40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 -- 172.21.15.90:0/2890994672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 msgr2=0x7f37cc0ff880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 0x7f37cc0ff880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 -- 172.21.15.90:0/2890994672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f37c80095f0 con 0x7f37cc1007d0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1c69700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 0x7f37cc0ff880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d246a700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc19df40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d1468700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f37c80095c0 tx=0x7f37c800f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37c801d070 con 0x7f37cc1007d0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f37cc19ec90 con 0x7f37cc1007d0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f37c800fb80 con 0x7f37cc1007d0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f37c8017650 con 0x7f37cc1007d0 2024-04-06T14:04:08.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.633+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37cc19f110 con 0x7f37cc1007d0 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.634+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f37cc107100 con 0x7f37cc1007d0 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.635+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f37c800fcf0 con 0x7f37cc1007d0 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.636+0000 7f37beffd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 0x7f37b80a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.636+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f37c80cddb0 con 0x7f37cc1007d0 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.636+0000 7f37d1c69700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 0x7f37b80a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:08.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.636+0000 7f37d1c69700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 0x7f37b80a3670 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f37c0005fd0 tx=0x7f37c0005f00 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:08.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.639+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f37c8097620 con 0x7f37cc1007d0 2024-04-06T14:04:08.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:08 smithi090 ceph-mon[43420]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:08 smithi090 ceph-mon[43420]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:08 smithi090 ceph-mon[43420]: pgmap v223: 97 pgs: 97 active+clean; 580 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 286 B/s rd, 3.2 KiB/s wr, 11 op/s 2024-04-06T14:04:08.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.880+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f37cc067e70 con 0x7f37cc1007d0 2024-04-06T14:04:08.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.880+0000 7f37beffd700 1 -- 172.21.15.90:0/2890994672 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f37c809ac40 con 0x7f37cc1007d0 2024-04-06T14:04:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:08.880 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:04:08.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 msgr2=0x7f37b80a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 0x7f37b80a3670 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f37c0005fd0 tx=0x7f37c0005f00 comp rx=0 tx=0).stop 2024-04-06T14:04:08.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 msgr2=0x7f37cc19b8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:08.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f37c80095c0 tx=0x7f37c800f690 comp rx=0 tx=0).stop 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 shutdown_connections 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f37b80a11b0 0x7f37b80a3670 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f37cc1011c0 0x7f37cc19df40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f37cc0ffe10 0x7f37cc0ff880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 --2- 172.21.15.90:0/2890994672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f37cc1007d0 0x7f37cc19b8f0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.882+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 >> 172.21.15.90:0/2890994672 conn(0x7f37cc0fa8c0 msgr2=0x7f37cc104240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.883+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 shutdown_connections 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:08.883+0000 7f37d3ecd700 1 -- 172.21.15.90:0/2890994672 wait complete. 2024-04-06T14:04:08.883 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:04:08.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:08 smithi134 ceph-mon[47549]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:08 smithi134 ceph-mon[47549]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:08 smithi134 ceph-mon[47549]: pgmap v223: 97 pgs: 97 active+clean; 580 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 286 B/s rd, 3.2 KiB/s wr, 11 op/s 2024-04-06T14:04:08.894 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:08.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:08 smithi105 ceph-mon[46969]: from='client.14757 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_0", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:08 smithi105 ceph-mon[46969]: from='client.14763 -' entity='client.admin' cmd=[{"prefix": "fs subvolume create", "vol_name": "cephfs", "sub_name": "sv_1", "group_name": "qa", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:08.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:08 smithi105 ceph-mon[46969]: pgmap v223: 97 pgs: 97 active+clean; 580 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 286 B/s rd, 3.2 KiB/s wr, 11 op/s 2024-04-06T14:04:09.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 -- 172.21.15.90:0/1165256340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 msgr2=0x7f46a81011f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/1165256340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a81011f0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f469c0099c0 tx=0x7f469c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 -- 172.21.15.90:0/1165256340 shutdown_connections 2024-04-06T14:04:09.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/1165256340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a8101730 0x7f46a8107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/1165256340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a80ffba0 0x7f46a80fffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/1165256340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a81011f0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 -- 172.21.15.90:0/1165256340 >> 172.21.15.90:0/1165256340 conn(0x7f46a80fb110 msgr2=0x7f46a80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 -- 172.21.15.90:0/1165256340 shutdown_connections 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.017+0000 7f46ae7a1700 1 -- 172.21.15.90:0/1165256340 wait complete. 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 Processor -- start 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 -- start start 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 0x7f46a8069d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 0x7f46a819ddd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a819e570 con 0x7f46a8100d90 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a819e6e0 con 0x7f46a8101730 2024-04-06T14:04:09.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46ae7a1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a819e8a0 con 0x7f46a80ffba0 2024-04-06T14:04:09.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46a77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46a77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34162/0 (socket says 172.21.15.90:34162) 2024-04-06T14:04:09.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46a77fe700 1 -- 172.21.15.90:0/3889399998 learned_addr learned my addr 172.21.15.90:0/3889399998 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:09.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46acd3e700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 0x7f46a819ddd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46a7fff700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 0x7f46a8069d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.018+0000 7f46a77fe700 1 -- 172.21.15.90:0/3889399998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 msgr2=0x7f46a8069d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a77fe700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 0x7f46a8069d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a77fe700 1 -- 172.21.15.90:0/3889399998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 msgr2=0x7f46a819ddd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a77fe700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 0x7f46a819ddd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a77fe700 1 -- 172.21.15.90:0/3889399998 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f469c0095f0 con 0x7f46a8100d90 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46acd3e700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 0x7f46a819ddd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a7fff700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 0x7f46a8069d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a77fe700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f469c0036f0 tx=0x7f469c003970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f469c01d070 con 0x7f46a8100d90 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46a819eb20 con 0x7f46a8100d90 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f469c022470 con 0x7f46a8100d90 2024-04-06T14:04:09.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f469c00f6e0 con 0x7f46a8100d90 2024-04-06T14:04:09.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.019+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46a819efd0 con 0x7f46a8100d90 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.020+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46a8106ef0 con 0x7f46a8100d90 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.021+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f469c022890 con 0x7f46a8100d90 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.021+0000 7f46a57fa700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 0x7f46900a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.021+0000 7f46a7fff700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 0x7f46900a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.021+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f469c0cdc70 con 0x7f46a8100d90 2024-04-06T14:04:09.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.022+0000 7f46a7fff700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 0x7f46900a3670 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4698005fd0 tx=0x7f4698005e40 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.026+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f469c05dab0 con 0x7f46a8100d90 2024-04-06T14:04:09.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.226+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f46a810b3b0 con 0x7f46a8100d90 2024-04-06T14:04:09.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.228+0000 7f46a57fa700 1 -- 172.21.15.90:0/3889399998 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7f469c0974e0 con 0x7f46a8100d90 2024-04-06T14:04:09.227 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:09.228 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:09.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 msgr2=0x7f46900a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 0x7f46900a3670 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f4698005fd0 tx=0x7f4698005e40 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 msgr2=0x7f46a819b7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f469c0036f0 tx=0x7f469c003970 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 shutdown_connections 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f46900a11b0 0x7f46900a3670 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a8101730 0x7f46a819ddd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a80ffba0 0x7f46a8069d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 --2- 172.21.15.90:0/3889399998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a8100d90 0x7f46a819b7e0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 >> 172.21.15.90:0/3889399998 conn(0x7f46a80fb110 msgr2=0x7f46a8104030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 shutdown_connections 2024-04-06T14:04:09.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.230+0000 7f46ae7a1700 1 -- 172.21.15.90:0/3889399998 wait complete. 2024-04-06T14:04:09.242 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2024-04-06T14:04:09.242 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-06T14:04:09.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 -- 172.21.15.90:0/201708573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 msgr2=0x7f7dc80731e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 --2- 172.21.15.90:0/201708573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc80731e0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f7db8009990 tx=0x7f7db8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 -- 172.21.15.90:0/201708573 shutdown_connections 2024-04-06T14:04:09.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 --2- 172.21.15.90:0/201708573 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 0x7f7dc810afd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 --2- 172.21.15.90:0/201708573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc80731e0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 --2- 172.21.15.90:0/201708573 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 0x7f7dc8073c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 -- 172.21.15.90:0/201708573 >> 172.21.15.90:0/201708573 conn(0x7f7dc80fc220 msgr2=0x7f7dc80fe640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 -- 172.21.15.90:0/201708573 shutdown_connections 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.364+0000 7f7dd015b700 1 -- 172.21.15.90:0/201708573 wait complete. 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 Processor -- start 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 -- start start 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 0x7f7dc8198dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 0x7f7dc81a0320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc81a08f0 con 0x7f7dc8074d90 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc81a0a30 con 0x7f7dc80737b0 2024-04-06T14:04:09.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dd015b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dc81a0bc0 con 0x7f7dc8108b60 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34184/0 (socket says 172.21.15.90:34184) 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 -- 172.21.15.90:0/3297590123 learned_addr learned my addr 172.21.15.90:0/3297590123 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcdef7700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 0x7f7dc8198dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dce6f8700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 0x7f7dc81a0320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 -- 172.21.15.90:0/3297590123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 msgr2=0x7f7dc81a0320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 0x7f7dc81a0320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 -- 172.21.15.90:0/3297590123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 msgr2=0x7f7dc8198dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 0x7f7dc8198dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.365+0000 7f7dcd6f6700 1 -- 172.21.15.90:0/3297590123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7db80095f0 con 0x7f7dc8074d90 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dcd6f6700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f7dc400d950 tx=0x7f7dc400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dc4009940 con 0x7f7dc8074d90 2024-04-06T14:04:09.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7dc81a0ea0 con 0x7f7dc8074d90 2024-04-06T14:04:09.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dc4010460 con 0x7f7dc8074d90 2024-04-06T14:04:09.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dc400f620 con 0x7f7dc8074d90 2024-04-06T14:04:09.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.366+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7dc81a1320 con 0x7f7dc8074d90 2024-04-06T14:04:09.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.367+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7dc8107970 con 0x7f7dc8074d90 2024-04-06T14:04:09.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.368+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f7dc40108d0 con 0x7f7dc8074d90 2024-04-06T14:04:09.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.368+0000 7f7dbeffd700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 0x7f7db40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.368+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f7dc40cc710 con 0x7f7dc8074d90 2024-04-06T14:04:09.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.368+0000 7f7dcdef7700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 0x7f7db40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.369+0000 7f7dcdef7700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 0x7f7db40a3670 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f7db800b5c0 tx=0x7f7db8003790 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.373+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7dc4095ef0 con 0x7f7dc8074d90 2024-04-06T14:04:09.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.571+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f7dc802cdc0 con 0x7f7db40a11b0 2024-04-06T14:04:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.581+0000 7f7dbeffd700 1 -- 172.21.15.90:0/3297590123 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+328352 (secure 0 0 0) 0x7f7dc802cdc0 con 0x7f7db40a11b0 2024-04-06T14:04:09.585 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 msgr2=0x7f7db40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 0x7f7db40a3670 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f7db800b5c0 tx=0x7f7db8003790 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 msgr2=0x7f7dc8199300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f7dc400d950 tx=0x7f7dc400dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 shutdown_connections 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7db40a11b0 0x7f7db40a3670 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7dc8108b60 0x7f7dc81a0320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7dc80737b0 0x7f7dc8198dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 --2- 172.21.15.90:0/3297590123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7dc8074d90 0x7f7dc8199300 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 >> 172.21.15.90:0/3297590123 conn(0x7f7dc80fc220 msgr2=0x7f7dc8104ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 shutdown_connections 2024-04-06T14:04:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.585+0000 7f7dd015b700 1 -- 172.21.15.90:0/3297590123 wait complete. 2024-04-06T14:04:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:04:09.596 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":223,"stamp":"2024-04-06T14:04:08.159973+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593886,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":138,"num_read_kb":121,"num_write":263,"num_write_kb":2961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":359232,"kb_used_data":10412,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124582592,"statfs":{"total":1151940427776,"available":1151572574208,"internally_reserved":0,"allocated":10661888,"data_stored":6789005,"data_compressed":107160,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3518,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":82,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.147102"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489854+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:04:01.489854+0000","last_peered":"2024-04-06T14:04:01.489854+0000","last_clean":"2024-04-06T14:04:01.489854+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:04:01.489854+0000","last_undegraded":"2024-04-06T14:04:01.489854+0000","last_fullsized":"2024-04-06T14:04:01.489854+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574988+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:04:02.574988+0000","last_peered":"2024-04-06T14:04:02.574988+0000","last_clean":"2024-04-06T14:04:02.574988+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:04:02.574988+0000","last_undegraded":"2024-04-06T14:04:02.574988+0000","last_fullsized":"2024-04-06T14:04:02.574988+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574769+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:04:02.574769+0000","last_peered":"2024-04-06T14:04:02.574769+0000","last_clean":"2024-04-06T14:04:02.574769+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:04:02.574769+0000","last_undegraded":"2024-04-06T14:04:02.574769+0000","last_fullsized":"2024-04-06T14:04:02.574769+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470214+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:04:01.470214+0000","last_peered":"2024-04-06T14:04:01.470214+0000","last_clean":"2024-04-06T14:04:01.470214+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:04:01.470214+0000","last_undegraded":"2024-04-06T14:04:01.470214+0000","last_fullsized":"2024-04-06T14:04:01.470214+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575085+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:04:02.575085+0000","last_peered":"2024-04-06T14:04:02.575085+0000","last_clean":"2024-04-06T14:04:02.575085+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:04:02.575085+0000","last_undegraded":"2024-04-06T14:04:02.575085+0000","last_fullsized":"2024-04-06T14:04:02.575085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436129+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:04:02.436129+0000","last_peered":"2024-04-06T14:04:02.436129+0000","last_clean":"2024-04-06T14:04:02.436129+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:04:02.436129+0000","last_undegraded":"2024-04-06T14:04:02.436129+0000","last_fullsized":"2024-04-06T14:04:02.436129+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225702+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:04:02.225702+0000","last_peered":"2024-04-06T14:04:02.225702+0000","last_clean":"2024-04-06T14:04:02.225702+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:04:02.225702+0000","last_undegraded":"2024-04-06T14:04:02.225702+0000","last_fullsized":"2024-04-06T14:04:02.225702+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575535+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:04:02.575535+0000","last_peered":"2024-04-06T14:04:02.575535+0000","last_clean":"2024-04-06T14:04:02.575535+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:04:02.575535+0000","last_undegraded":"2024-04-06T14:04:02.575535+0000","last_fullsized":"2024-04-06T14:04:02.575535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575974+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:04:02.575974+0000","last_peered":"2024-04-06T14:04:02.575974+0000","last_clean":"2024-04-06T14:04:02.575974+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:04:02.575974+0000","last_undegraded":"2024-04-06T14:04:02.575974+0000","last_fullsized":"2024-04-06T14:04:02.575974+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436412+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:04:02.436412+0000","last_peered":"2024-04-06T14:04:02.436412+0000","last_clean":"2024-04-06T14:04:02.436412+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:04:02.436412+0000","last_undegraded":"2024-04-06T14:04:02.436412+0000","last_fullsized":"2024-04-06T14:04:02.436412+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225803+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:04:02.225803+0000","last_peered":"2024-04-06T14:04:02.225803+0000","last_clean":"2024-04-06T14:04:02.225803+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:04:02.225803+0000","last_undegraded":"2024-04-06T14:04:02.225803+0000","last_fullsized":"2024-04-06T14:04:02.225803+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470202+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:04:01.470202+0000","last_peered":"2024-04-06T14:04:01.470202+0000","last_clean":"2024-04-06T14:04:01.470202+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:04:01.470202+0000","last_undegraded":"2024-04-06T14:04:01.470202+0000","last_fullsized":"2024-04-06T14:04:01.470202+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575159+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:04:02.575159+0000","last_peered":"2024-04-06T14:04:02.575159+0000","last_clean":"2024-04-06T14:04:02.575159+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:04:02.575159+0000","last_undegraded":"2024-04-06T14:04:02.575159+0000","last_fullsized":"2024-04-06T14:04:02.575159+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574684+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:04:02.574684+0000","last_peered":"2024-04-06T14:04:02.574684+0000","last_clean":"2024-04-06T14:04:02.574684+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:04:02.574684+0000","last_undegraded":"2024-04-06T14:04:02.574684+0000","last_fullsized":"2024-04-06T14:04:02.574684+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574459+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:04:02.574459+0000","last_peered":"2024-04-06T14:04:02.574459+0000","last_clean":"2024-04-06T14:04:02.574459+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:04:02.574459+0000","last_undegraded":"2024-04-06T14:04:02.574459+0000","last_fullsized":"2024-04-06T14:04:02.574459+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575094+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:04:02.575094+0000","last_peered":"2024-04-06T14:04:02.575094+0000","last_clean":"2024-04-06T14:04:02.575094+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:04:02.575094+0000","last_undegraded":"2024-04-06T14:04:02.575094+0000","last_fullsized":"2024-04-06T14:04:02.575094+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489865+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:04:01.489865+0000","last_peered":"2024-04-06T14:04:01.489865+0000","last_clean":"2024-04-06T14:04:01.489865+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:04:01.489865+0000","last_undegraded":"2024-04-06T14:04:01.489865+0000","last_fullsized":"2024-04-06T14:04:01.489865+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437817+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:04:02.437817+0000","last_peered":"2024-04-06T14:04:02.437817+0000","last_clean":"2024-04-06T14:04:02.437817+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:04:02.437817+0000","last_undegraded":"2024-04-06T14:04:02.437817+0000","last_fullsized":"2024-04-06T14:04:02.437817+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672247+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:04:01.672247+0000","last_peered":"2024-04-06T14:04:01.672247+0000","last_clean":"2024-04-06T14:04:01.672247+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:04:01.672247+0000","last_undegraded":"2024-04-06T14:04:01.672247+0000","last_fullsized":"2024-04-06T14:04:01.672247+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":21,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513680+0000","last_change":"2024-04-06T14:04:01.028407+0000","last_active":"2024-04-06T14:04:05.513680+0000","last_peered":"2024-04-06T14:04:05.513680+0000","last_clean":"2024-04-06T14:04:05.513680+0000","last_became_active":"2024-04-06T14:04:01.028036+0000","last_became_peered":"2024-04-06T14:04:01.028036+0000","last_unstale":"2024-04-06T14:04:05.513680+0000","last_undegraded":"2024-04-06T14:04:05.513680+0000","last_fullsized":"2024-04-06T14:04:05.513680+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:08:53.486282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575627+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:04:02.575627+0000","last_peered":"2024-04-06T14:04:02.575627+0000","last_clean":"2024-04-06T14:04:02.575627+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:04:02.575627+0000","last_undegraded":"2024-04-06T14:04:02.575627+0000","last_fullsized":"2024-04-06T14:04:02.575627+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575635+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:04:02.575635+0000","last_peered":"2024-04-06T14:04:02.575635+0000","last_clean":"2024-04-06T14:04:02.575635+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:04:02.575635+0000","last_undegraded":"2024-04-06T14:04:02.575635+0000","last_fullsized":"2024-04-06T14:04:02.575635+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575409+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:04:02.575409+0000","last_peered":"2024-04-06T14:04:02.575409+0000","last_clean":"2024-04-06T14:04:02.575409+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:04:02.575409+0000","last_undegraded":"2024-04-06T14:04:02.575409+0000","last_fullsized":"2024-04-06T14:04:02.575409+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575420+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:04:02.575420+0000","last_peered":"2024-04-06T14:04:02.575420+0000","last_clean":"2024-04-06T14:04:02.575420+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:04:02.575420+0000","last_undegraded":"2024-04-06T14:04:02.575420+0000","last_fullsized":"2024-04-06T14:04:02.575420+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575221+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:04:02.575221+0000","last_peered":"2024-04-06T14:04:02.575221+0000","last_clean":"2024-04-06T14:04:02.575221+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:04:02.575221+0000","last_undegraded":"2024-04-06T14:04:02.575221+0000","last_fullsized":"2024-04-06T14:04:02.575221+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575477+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:04:02.575477+0000","last_peered":"2024-04-06T14:04:02.575477+0000","last_clean":"2024-04-06T14:04:02.575477+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:04:02.575477+0000","last_undegraded":"2024-04-06T14:04:02.575477+0000","last_fullsized":"2024-04-06T14:04:02.575477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574918+0000","last_change":"2024-04-06T14:04:01.457311+0000","last_active":"2024-04-06T14:04:02.574918+0000","last_peered":"2024-04-06T14:04:02.574918+0000","last_clean":"2024-04-06T14:04:02.574918+0000","last_became_active":"2024-04-06T14:04:01.456954+0000","last_became_peered":"2024-04-06T14:04:01.456954+0000","last_unstale":"2024-04-06T14:04:02.574918+0000","last_undegraded":"2024-04-06T14:04:02.574918+0000","last_fullsized":"2024-04-06T14:04:02.574918+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:22:59.524995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574766+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:04:02.574766+0000","last_peered":"2024-04-06T14:04:02.574766+0000","last_clean":"2024-04-06T14:04:02.574766+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:04:02.574766+0000","last_undegraded":"2024-04-06T14:04:02.574766+0000","last_fullsized":"2024-04-06T14:04:02.574766+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575517+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:04:02.575517+0000","last_peered":"2024-04-06T14:04:02.575517+0000","last_clean":"2024-04-06T14:04:02.575517+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:04:02.575517+0000","last_undegraded":"2024-04-06T14:04:02.575517+0000","last_fullsized":"2024-04-06T14:04:02.575517+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436039+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:04:02.436039+0000","last_peered":"2024-04-06T14:04:02.436039+0000","last_clean":"2024-04-06T14:04:02.436039+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:04:02.436039+0000","last_undegraded":"2024-04-06T14:04:02.436039+0000","last_fullsized":"2024-04-06T14:04:02.436039+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574780+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:04:02.574780+0000","last_peered":"2024-04-06T14:04:02.574780+0000","last_clean":"2024-04-06T14:04:02.574780+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:04:02.574780+0000","last_undegraded":"2024-04-06T14:04:02.574780+0000","last_fullsized":"2024-04-06T14:04:02.574780+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.439215+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:04:01.439215+0000","last_peered":"2024-04-06T14:04:01.439215+0000","last_clean":"2024-04-06T14:04:01.439215+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:04:01.439215+0000","last_undegraded":"2024-04-06T14:04:01.439215+0000","last_fullsized":"2024-04-06T14:04:01.439215+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575621+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:04:02.575621+0000","last_peered":"2024-04-06T14:04:02.575621+0000","last_clean":"2024-04-06T14:04:02.575621+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:04:02.575621+0000","last_undegraded":"2024-04-06T14:04:02.575621+0000","last_fullsized":"2024-04-06T14:04:02.575621+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574562+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:04:02.574562+0000","last_peered":"2024-04-06T14:04:02.574562+0000","last_clean":"2024-04-06T14:04:02.574562+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:04:02.574562+0000","last_undegraded":"2024-04-06T14:04:02.574562+0000","last_fullsized":"2024-04-06T14:04:02.574562+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437700+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:04:02.437700+0000","last_peered":"2024-04-06T14:04:02.437700+0000","last_clean":"2024-04-06T14:04:02.437700+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:04:02.437700+0000","last_undegraded":"2024-04-06T14:04:02.437700+0000","last_fullsized":"2024-04-06T14:04:02.437700+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436356+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:04:02.436356+0000","last_peered":"2024-04-06T14:04:02.436356+0000","last_clean":"2024-04-06T14:04:02.436356+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:04:02.436356+0000","last_undegraded":"2024-04-06T14:04:02.436356+0000","last_fullsized":"2024-04-06T14:04:02.436356+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575670+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:04:02.575670+0000","last_peered":"2024-04-06T14:04:02.575670+0000","last_clean":"2024-04-06T14:04:02.575670+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:04:02.575670+0000","last_undegraded":"2024-04-06T14:04:02.575670+0000","last_fullsized":"2024-04-06T14:04:02.575670+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507363+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:04:04.507363+0000","last_peered":"2024-04-06T14:04:04.507363+0000","last_clean":"2024-04-06T14:04:04.507363+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:04:04.507363+0000","last_undegraded":"2024-04-06T14:04:04.507363+0000","last_fullsized":"2024-04-06T14:04:04.507363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575323+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:04:02.575323+0000","last_peered":"2024-04-06T14:04:02.575323+0000","last_clean":"2024-04-06T14:04:02.575323+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:04:02.575323+0000","last_undegraded":"2024-04-06T14:04:02.575323+0000","last_fullsized":"2024-04-06T14:04:02.575323+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"80'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575059+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:04:02.575059+0000","last_peered":"2024-04-06T14:04:02.575059+0000","last_clean":"2024-04-06T14:04:02.575059+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:04:02.575059+0000","last_undegraded":"2024-04-06T14:04:02.575059+0000","last_fullsized":"2024-04-06T14:04:02.575059+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575352+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:04:02.575352+0000","last_peered":"2024-04-06T14:04:02.575352+0000","last_clean":"2024-04-06T14:04:02.575352+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:04:02.575352+0000","last_undegraded":"2024-04-06T14:04:02.575352+0000","last_fullsized":"2024-04-06T14:04:02.575352+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'6","reported_seq":18,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436258+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:04:02.436258+0000","last_peered":"2024-04-06T14:04:02.436258+0000","last_clean":"2024-04-06T14:04:02.436258+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:04:02.436258+0000","last_undegraded":"2024-04-06T14:04:02.436258+0000","last_fullsized":"2024-04-06T14:04:02.436258+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575168+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:04:02.575168+0000","last_peered":"2024-04-06T14:04:02.575168+0000","last_clean":"2024-04-06T14:04:02.575168+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:04:02.575168+0000","last_undegraded":"2024-04-06T14:04:02.575168+0000","last_fullsized":"2024-04-06T14:04:02.575168+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":19,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513503+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:04:05.513503+0000","last_peered":"2024-04-06T14:04:05.513503+0000","last_clean":"2024-04-06T14:04:05.513503+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:04:05.513503+0000","last_undegraded":"2024-04-06T14:04:05.513503+0000","last_fullsized":"2024-04-06T14:04:05.513503+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574857+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:04:02.574857+0000","last_peered":"2024-04-06T14:04:02.574857+0000","last_clean":"2024-04-06T14:04:02.574857+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:04:02.574857+0000","last_undegraded":"2024-04-06T14:04:02.574857+0000","last_fullsized":"2024-04-06T14:04:02.574857+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574877+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:04:02.574877+0000","last_peered":"2024-04-06T14:04:02.574877+0000","last_clean":"2024-04-06T14:04:02.574877+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:04:02.574877+0000","last_undegraded":"2024-04-06T14:04:02.574877+0000","last_fullsized":"2024-04-06T14:04:02.574877+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225762+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:04:02.225762+0000","last_peered":"2024-04-06T14:04:02.225762+0000","last_clean":"2024-04-06T14:04:02.225762+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:04:02.225762+0000","last_undegraded":"2024-04-06T14:04:02.225762+0000","last_fullsized":"2024-04-06T14:04:02.225762+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512240+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:04:05.512240+0000","last_peered":"2024-04-06T14:04:05.512240+0000","last_clean":"2024-04-06T14:04:05.512240+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:04:05.512240+0000","last_undegraded":"2024-04-06T14:04:05.512240+0000","last_fullsized":"2024-04-06T14:04:05.512240+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225612+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:04:02.225612+0000","last_peered":"2024-04-06T14:04:02.225612+0000","last_clean":"2024-04-06T14:04:02.225612+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:04:02.225612+0000","last_undegraded":"2024-04-06T14:04:02.225612+0000","last_fullsized":"2024-04-06T14:04:02.225612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"80'8","reported_seq":20,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574985+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:04:02.574985+0000","last_peered":"2024-04-06T14:04:02.574985+0000","last_clean":"2024-04-06T14:04:02.574985+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:04:02.574985+0000","last_undegraded":"2024-04-06T14:04:02.574985+0000","last_fullsized":"2024-04-06T14:04:02.574985+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574613+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:04:02.574613+0000","last_peered":"2024-04-06T14:04:02.574613+0000","last_clean":"2024-04-06T14:04:02.574613+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:04:02.574613+0000","last_undegraded":"2024-04-06T14:04:02.574613+0000","last_fullsized":"2024-04-06T14:04:02.574613+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575394+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:04:02.575394+0000","last_peered":"2024-04-06T14:04:02.575394+0000","last_clean":"2024-04-06T14:04:02.575394+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:04:02.575394+0000","last_undegraded":"2024-04-06T14:04:02.575394+0000","last_fullsized":"2024-04-06T14:04:02.575394+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489844+0000","last_change":"2024-04-06T14:04:01.455152+0000","last_active":"2024-04-06T14:04:01.489844+0000","last_peered":"2024-04-06T14:04:01.489844+0000","last_clean":"2024-04-06T14:04:01.489844+0000","last_became_active":"2024-04-06T14:04:01.454827+0000","last_became_peered":"2024-04-06T14:04:01.454827+0000","last_unstale":"2024-04-06T14:04:01.489844+0000","last_undegraded":"2024-04-06T14:04:01.489844+0000","last_fullsized":"2024-04-06T14:04:01.489844+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:55:26.359159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.438338+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:04:01.438338+0000","last_peered":"2024-04-06T14:04:01.438338+0000","last_clean":"2024-04-06T14:04:01.438338+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:04:01.438338+0000","last_undegraded":"2024-04-06T14:04:01.438338+0000","last_fullsized":"2024-04-06T14:04:01.438338+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436512+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:04:02.436512+0000","last_peered":"2024-04-06T14:04:02.436512+0000","last_clean":"2024-04-06T14:04:02.436512+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:04:02.436512+0000","last_undegraded":"2024-04-06T14:04:02.436512+0000","last_fullsized":"2024-04-06T14:04:02.436512+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576581+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:04:02.576581+0000","last_peered":"2024-04-06T14:04:02.576581+0000","last_clean":"2024-04-06T14:04:02.576581+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:04:02.576581+0000","last_undegraded":"2024-04-06T14:04:02.576581+0000","last_fullsized":"2024-04-06T14:04:02.576581+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436536+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:04:02.436536+0000","last_peered":"2024-04-06T14:04:02.436536+0000","last_clean":"2024-04-06T14:04:02.436536+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:04:02.436536+0000","last_undegraded":"2024-04-06T14:04:02.436536+0000","last_fullsized":"2024-04-06T14:04:02.436536+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575047+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:04:02.575047+0000","last_peered":"2024-04-06T14:04:02.575047+0000","last_clean":"2024-04-06T14:04:02.575047+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:04:02.575047+0000","last_undegraded":"2024-04-06T14:04:02.575047+0000","last_fullsized":"2024-04-06T14:04:02.575047+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574547+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:04:02.574547+0000","last_peered":"2024-04-06T14:04:02.574547+0000","last_clean":"2024-04-06T14:04:02.574547+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:04:02.574547+0000","last_undegraded":"2024-04-06T14:04:02.574547+0000","last_fullsized":"2024-04-06T14:04:02.574547+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575189+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:04:02.575189+0000","last_peered":"2024-04-06T14:04:02.575189+0000","last_clean":"2024-04-06T14:04:02.575189+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:04:02.575189+0000","last_undegraded":"2024-04-06T14:04:02.575189+0000","last_fullsized":"2024-04-06T14:04:02.575189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.508080+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:04:04.508080+0000","last_peered":"2024-04-06T14:04:04.508080+0000","last_clean":"2024-04-06T14:04:04.508080+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:04:04.508080+0000","last_undegraded":"2024-04-06T14:04:04.508080+0000","last_fullsized":"2024-04-06T14:04:04.508080+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":333,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.451452+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:04:01.451452+0000","last_peered":"2024-04-06T14:04:01.451452+0000","last_clean":"2024-04-06T14:04:01.451452+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:04:01.451452+0000","last_undegraded":"2024-04-06T14:04:01.451452+0000","last_fullsized":"2024-04-06T14:04:01.451452+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574458+0000","last_change":"2024-04-06T14:04:01.457132+0000","last_active":"2024-04-06T14:04:02.574458+0000","last_peered":"2024-04-06T14:04:02.574458+0000","last_clean":"2024-04-06T14:04:02.574458+0000","last_became_active":"2024-04-06T14:04:01.456782+0000","last_became_peered":"2024-04-06T14:04:01.456782+0000","last_unstale":"2024-04-06T14:04:02.574458+0000","last_undegraded":"2024-04-06T14:04:02.574458+0000","last_fullsized":"2024-04-06T14:04:02.574458+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:34:37.759113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512264+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:04:05.512264+0000","last_peered":"2024-04-06T14:04:05.512264+0000","last_clean":"2024-04-06T14:04:05.512264+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:04:05.512264+0000","last_undegraded":"2024-04-06T14:04:05.512264+0000","last_fullsized":"2024-04-06T14:04:05.512264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575260+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:04:02.575260+0000","last_peered":"2024-04-06T14:04:02.575260+0000","last_clean":"2024-04-06T14:04:02.575260+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:04:02.575260+0000","last_undegraded":"2024-04-06T14:04:02.575260+0000","last_fullsized":"2024-04-06T14:04:02.575260+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574508+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:04:02.574508+0000","last_peered":"2024-04-06T14:04:02.574508+0000","last_clean":"2024-04-06T14:04:02.574508+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:04:02.574508+0000","last_undegraded":"2024-04-06T14:04:02.574508+0000","last_fullsized":"2024-04-06T14:04:02.574508+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513917+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:04:05.513917+0000","last_peered":"2024-04-06T14:04:05.513917+0000","last_clean":"2024-04-06T14:04:05.513917+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:04:05.513917+0000","last_undegraded":"2024-04-06T14:04:05.513917+0000","last_fullsized":"2024-04-06T14:04:05.513917+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575544+0000","last_change":"2024-04-06T14:04:01.461874+0000","last_active":"2024-04-06T14:04:02.575544+0000","last_peered":"2024-04-06T14:04:02.575544+0000","last_clean":"2024-04-06T14:04:02.575544+0000","last_became_active":"2024-04-06T14:04:01.461652+0000","last_became_peered":"2024-04-06T14:04:01.461652+0000","last_unstale":"2024-04-06T14:04:02.575544+0000","last_undegraded":"2024-04-06T14:04:02.575544+0000","last_fullsized":"2024-04-06T14:04:02.575544+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:01:58.862215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507264+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:04:04.507264+0000","last_peered":"2024-04-06T14:04:04.507264+0000","last_clean":"2024-04-06T14:04:04.507264+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:04:04.507264+0000","last_undegraded":"2024-04-06T14:04:04.507264+0000","last_fullsized":"2024-04-06T14:04:04.507264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575075+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:04:02.575075+0000","last_peered":"2024-04-06T14:04:02.575075+0000","last_clean":"2024-04-06T14:04:02.575075+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:04:02.575075+0000","last_undegraded":"2024-04-06T14:04:02.575075+0000","last_fullsized":"2024-04-06T14:04:02.575075+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507554+0000","last_change":"2024-04-06T14:04:01.035529+0000","last_active":"2024-04-06T14:04:04.507554+0000","last_peered":"2024-04-06T14:04:04.507554+0000","last_clean":"2024-04-06T14:04:04.507554+0000","last_became_active":"2024-04-06T14:04:01.035205+0000","last_became_peered":"2024-04-06T14:04:01.035205+0000","last_unstale":"2024-04-06T14:04:04.507554+0000","last_undegraded":"2024-04-06T14:04:04.507554+0000","last_fullsized":"2024-04-06T14:04:04.507554+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:59:17.605780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574775+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:04:02.574775+0000","last_peered":"2024-04-06T14:04:02.574775+0000","last_clean":"2024-04-06T14:04:02.574775+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:04:02.574775+0000","last_undegraded":"2024-04-06T14:04:02.574775+0000","last_fullsized":"2024-04-06T14:04:02.574775+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574687+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:04:02.574687+0000","last_peered":"2024-04-06T14:04:02.574687+0000","last_clean":"2024-04-06T14:04:02.574687+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:04:02.574687+0000","last_undegraded":"2024-04-06T14:04:02.574687+0000","last_fullsized":"2024-04-06T14:04:02.574687+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489773+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:04:01.489773+0000","last_peered":"2024-04-06T14:04:01.489773+0000","last_clean":"2024-04-06T14:04:01.489773+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:04:01.489773+0000","last_undegraded":"2024-04-06T14:04:01.489773+0000","last_fullsized":"2024-04-06T14:04:01.489773+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672174+0000","last_change":"2024-04-06T14:04:01.032612+0000","last_active":"2024-04-06T14:04:01.672174+0000","last_peered":"2024-04-06T14:04:01.672174+0000","last_clean":"2024-04-06T14:04:01.672174+0000","last_became_active":"2024-04-06T14:04:01.032115+0000","last_became_peered":"2024-04-06T14:04:01.032115+0000","last_unstale":"2024-04-06T14:04:01.672174+0000","last_undegraded":"2024-04-06T14:04:01.672174+0000","last_fullsized":"2024-04-06T14:04:01.672174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:40.607887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575432+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:04:02.575432+0000","last_peered":"2024-04-06T14:04:02.575432+0000","last_clean":"2024-04-06T14:04:02.575432+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:04:02.575432+0000","last_undegraded":"2024-04-06T14:04:02.575432+0000","last_fullsized":"2024-04-06T14:04:02.575432+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575163+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:04:02.575163+0000","last_peered":"2024-04-06T14:04:02.575163+0000","last_clean":"2024-04-06T14:04:02.575163+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:04:02.575163+0000","last_undegraded":"2024-04-06T14:04:02.575163+0000","last_fullsized":"2024-04-06T14:04:02.575163+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225742+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:04:02.225742+0000","last_peered":"2024-04-06T14:04:02.225742+0000","last_clean":"2024-04-06T14:04:02.225742+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:04:02.225742+0000","last_undegraded":"2024-04-06T14:04:02.225742+0000","last_fullsized":"2024-04-06T14:04:02.225742+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513714+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:04:05.513714+0000","last_peered":"2024-04-06T14:04:05.513714+0000","last_clean":"2024-04-06T14:04:05.513714+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:04:05.513714+0000","last_undegraded":"2024-04-06T14:04:05.513714+0000","last_fullsized":"2024-04-06T14:04:05.513714+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225824+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:04:02.225824+0000","last_peered":"2024-04-06T14:04:02.225824+0000","last_clean":"2024-04-06T14:04:02.225824+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:04:02.225824+0000","last_undegraded":"2024-04-06T14:04:02.225824+0000","last_fullsized":"2024-04-06T14:04:02.225824+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574856+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:04:02.574856+0000","last_peered":"2024-04-06T14:04:02.574856+0000","last_clean":"2024-04-06T14:04:02.574856+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:04:02.574856+0000","last_undegraded":"2024-04-06T14:04:02.574856+0000","last_fullsized":"2024-04-06T14:04:02.574856+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225783+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:04:02.225783+0000","last_peered":"2024-04-06T14:04:02.225783+0000","last_clean":"2024-04-06T14:04:02.225783+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:04:02.225783+0000","last_undegraded":"2024-04-06T14:04:02.225783+0000","last_fullsized":"2024-04-06T14:04:02.225783+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":14,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.467592+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:04:01.467592+0000","last_peered":"2024-04-06T14:04:01.467592+0000","last_clean":"2024-04-06T14:04:01.467592+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:04:01.467592+0000","last_undegraded":"2024-04-06T14:04:01.467592+0000","last_fullsized":"2024-04-06T14:04:01.467592+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225872+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:04:02.225872+0000","last_peered":"2024-04-06T14:04:02.225872+0000","last_clean":"2024-04-06T14:04:02.225872+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:04:02.225872+0000","last_undegraded":"2024-04-06T14:04:02.225872+0000","last_fullsized":"2024-04-06T14:04:02.225872+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489791+0000","last_change":"2024-04-06T14:04:01.454256+0000","last_active":"2024-04-06T14:04:01.489791+0000","last_peered":"2024-04-06T14:04:01.489791+0000","last_clean":"2024-04-06T14:04:01.489791+0000","last_became_active":"2024-04-06T14:04:01.452355+0000","last_became_peered":"2024-04-06T14:04:01.452355+0000","last_unstale":"2024-04-06T14:04:01.489791+0000","last_undegraded":"2024-04-06T14:04:01.489791+0000","last_fullsized":"2024-04-06T14:04:01.489791+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:34:42.131742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672108+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:04:01.672108+0000","last_peered":"2024-04-06T14:04:01.672108+0000","last_clean":"2024-04-06T14:04:01.672108+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:04:01.672108+0000","last_undegraded":"2024-04-06T14:04:01.672108+0000","last_fullsized":"2024-04-06T14:04:01.672108+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672134+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:04:01.672134+0000","last_peered":"2024-04-06T14:04:01.672134+0000","last_clean":"2024-04-06T14:04:01.672134+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:04:01.672134+0000","last_undegraded":"2024-04-06T14:04:01.672134+0000","last_fullsized":"2024-04-06T14:04:01.672134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574869+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:04:02.574869+0000","last_peered":"2024-04-06T14:04:02.574869+0000","last_clean":"2024-04-06T14:04:02.574869+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:04:02.574869+0000","last_undegraded":"2024-04-06T14:04:02.574869+0000","last_fullsized":"2024-04-06T14:04:02.574869+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575055+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:04:02.575055+0000","last_peered":"2024-04-06T14:04:02.575055+0000","last_clean":"2024-04-06T14:04:02.575055+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:04:02.575055+0000","last_undegraded":"2024-04-06T14:04:02.575055+0000","last_fullsized":"2024-04-06T14:04:02.575055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489833+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:04:01.489833+0000","last_peered":"2024-04-06T14:04:01.489833+0000","last_clean":"2024-04-06T14:04:01.489833+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:04:01.489833+0000","last_undegraded":"2024-04-06T14:04:01.489833+0000","last_fullsized":"2024-04-06T14:04:01.489833+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672268+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:04:01.672268+0000","last_peered":"2024-04-06T14:04:01.672268+0000","last_clean":"2024-04-06T14:04:01.672268+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:04:01.672268+0000","last_undegraded":"2024-04-06T14:04:01.672268+0000","last_fullsized":"2024-04-06T14:04:01.672268+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575443+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:04:02.575443+0000","last_peered":"2024-04-06T14:04:02.575443+0000","last_clean":"2024-04-06T14:04:02.575443+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:04:02.575443+0000","last_undegraded":"2024-04-06T14:04:02.575443+0000","last_fullsized":"2024-04-06T14:04:02.575443+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574980+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:04:02.574980+0000","last_peered":"2024-04-06T14:04:02.574980+0000","last_clean":"2024-04-06T14:04:02.574980+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:04:02.574980+0000","last_undegraded":"2024-04-06T14:04:02.574980+0000","last_fullsized":"2024-04-06T14:04:02.574980+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574778+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:04:02.574778+0000","last_peered":"2024-04-06T14:04:02.574778+0000","last_clean":"2024-04-06T14:04:02.574778+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:04:02.574778+0000","last_undegraded":"2024-04-06T14:04:02.574778+0000","last_fullsized":"2024-04-06T14:04:02.574778+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575547+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:04:02.575547+0000","last_peered":"2024-04-06T14:04:02.575547+0000","last_clean":"2024-04-06T14:04:02.575547+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:04:02.575547+0000","last_undegraded":"2024-04-06T14:04:02.575547+0000","last_fullsized":"2024-04-06T14:04:02.575547+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575324+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:04:02.575324+0000","last_peered":"2024-04-06T14:04:02.575324+0000","last_clean":"2024-04-06T14:04:02.575324+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:04:02.575324+0000","last_undegraded":"2024-04-06T14:04:02.575324+0000","last_fullsized":"2024-04-06T14:04:02.575324+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489822+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:04:01.489822+0000","last_peered":"2024-04-06T14:04:01.489822+0000","last_clean":"2024-04-06T14:04:01.489822+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:04:01.489822+0000","last_undegraded":"2024-04-06T14:04:01.489822+0000","last_fullsized":"2024-04-06T14:04:01.489822+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":376832,"data_stored":4132576,"data_compressed":246766,"data_compressed_allocated":315392,"data_compressed_original":4096000,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3518,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":82,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":11253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70,"ondisk_log_size":70,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514507,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":868352,"data_stored":417921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76300000000000001}]}]},{"osd":10,"up_from":68,"seq":292057776141,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":933888,"data_stored":1009134,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2,3,4,5,6,7,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41999999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":9,"up_from":63,"seq":270582939665,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":418123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.45700000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.75700000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.621},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.72699999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.495}]},{"osd":7,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.746}]},{"osd":8,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.67200000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68600000000000005}]}]},{"osd":8,"up_from":56,"seq":240518168596,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27872,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717280,"statfs":{"total":95995035648,"available":95966494720,"internally_reserved":0,"allocated":868352,"data_stored":417612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.48799999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56499999999999995}]},{"osd":2,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.57899999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.628},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53700000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]}]},{"osd":7,"up_from":49,"seq":210453397528,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32016,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713136,"statfs":{"total":95995035648,"available":95962251264,"internally_reserved":0,"allocated":925696,"data_stored":1009072,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.629}]},{"osd":4,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.505}]},{"osd":5,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.745}]},{"osd":6,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66800000000000004}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.78600000000000003}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.70199999999999996}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.754}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83299999999999996}]}]},{"osd":6,"up_from":45,"seq":193273528347,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":844,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":864256,"data_stored":417339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.315},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.39600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51000000000000001,"15min":0.502},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46899999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.504}]},{"osd":5,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83599999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.63600000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.62}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.57099999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.53900000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":876544,"data_stored":418903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46800000000000003,"15min":0.46999999999999997},"min":{"1min":0.36099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.47599999999999998,"15min":0.46899999999999997},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Apr 6 14:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.51800000000000002,"15min":0.496},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75600000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52400000000000002,"15min":0.50700000000000001},"min":{"1min":0.39900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70399999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Apr 6 14:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.48599999999999999,"15min":0.47999999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77200000000000002,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.42299999999999999,"15min":0.41099999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79200000000000004,"5min":0.877,"15min":0.877},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56899999999999995,"15min":0.55700000000000005},"min":{"1min":0.48499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.432},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.38100000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.74399999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.69199999999999995}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27868,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717284,"statfs":{"total":95995035648,"available":95966498816,"internally_reserved":0,"allocated":868352,"data_stored":417429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51700000000000002},"min":{"1min":0.32700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.67200000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63800000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.375}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.502,"15min":0.5},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.498}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.443,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46700000000000003,"15min":0.46899999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70799999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.56499999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.48799999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":420205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.47299999999999998,"15min":0.45300000000000001},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45600000000000002,"15min":0.439},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53500000000000003,"15min":0.51700000000000002},"min":{"1min":0.438,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71199999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51900000000000002,"15min":0.51100000000000001},"min":{"1min":0.432,"5min":0.248,"15min":0.248},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.55200000000000005,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.42999999999999999},"min":{"1min":0.32700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.42399999999999999,"15min":0.41499999999999998},"min":{"1min":0.31900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60599999999999998,"15min":0.59999999999999998},"min":{"1min":0.44,"5min":0.372,"15min":0.372},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.75700000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.74299999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.79400000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71199999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81799999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36064,"kb_used_data":852,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93709088,"statfs":{"total":95995035648,"available":95958106112,"internally_reserved":0,"allocated":872448,"data_stored":417443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.33400000000000002},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55100000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.372},{"interface":"front","average":{"1min":0.379,"5min":0.33800000000000002,"15min":0.33100000000000002},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.36499999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.49299999999999999,"15min":0.48499999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.67400000000000004}]},{"osd":2,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.53100000000000003,"15min":0.52400000000000002},"min":{"1min":0.41299999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.442,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54500000000000004,"15min":0.54300000000000004},"min":{"1min":0.41299999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.46700000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.626},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.68999999999999995}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.438}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70799999999999996}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71599999999999997}]}]},{"osd":4,"up_from":33,"seq":141733920803,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":876544,"data_stored":417935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.38,"15min":0.371},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.497}]},{"osd":2,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.53900000000000003,"15min":0.53000000000000003},"min":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.747}]},{"osd":3,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.50600000000000001,"15min":0.502},"min":{"1min":0.34899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.46899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.55600000000000005,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.48299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.81799999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.621,"5min":0.52400000000000002,"15min":0.50800000000000001},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.621},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.65100000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.694,"5min":1.694,"15min":1.694},"last":0.68200000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:03:53 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.66700000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81100000000000005}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32028,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713124,"statfs":{"total":95995035648,"available":95962238976,"internally_reserved":0,"allocated":925696,"data_stored":1007889,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.49299999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.48999999999999999,"15min":0.47699999999999998},"min":{"1min":0.36599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.52400000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51100000000000001,"15min":0.497},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.748},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.499,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.83799999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.38300000000000001,"15min":0.37},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.317,"5min":0.223,"15min":0.223},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.52900000000000003,"15min":0.51600000000000001},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57199999999999995,"15min":0.56200000000000006},"min":{"1min":0.45400000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83199999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.627,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.876,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.76200000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.873}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.81999999999999995}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":2866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:04:09.603 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 -- 172.21.15.90:0/377378346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4102810 msgr2=0x7f64a41090c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 --2- 172.21.15.90:0/377378346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4102810 0x7f64a41090c0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f649c00b0d0 tx=0x7f649c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 -- 172.21.15.90:0/377378346 shutdown_connections 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 --2- 172.21.15.90:0/377378346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4102810 0x7f64a41090c0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 --2- 172.21.15.90:0/377378346 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4100c80 0x7f64a41010a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 --2- 172.21.15.90:0/377378346 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f64a4101e70 0x7f64a41022d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.758+0000 7f64ab649700 1 -- 172.21.15.90:0/377378346 >> 172.21.15.90:0/377378346 conn(0x7f64a40fc210 msgr2=0x7f64a40fe630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- 172.21.15.90:0/377378346 shutdown_connections 2024-04-06T14:04:09.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- 172.21.15.90:0/377378346 wait complete. 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 Processor -- start 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- start start 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f64a4101e70 0x7f64a4197200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 0x7f64a419e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64a419e7e0 con 0x7f64a4100c80 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64a419e950 con 0x7f64a4102810 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64ab649700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64a419eb10 con 0x7f64a4101e70 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64a93e5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64a93e5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34198/0 (socket says 172.21.15.90:34198) 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.759+0000 7f64a93e5700 1 -- 172.21.15.90:0/349407425 learned_addr learned my addr 172.21.15.90:0/349407425 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a9be6700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 0x7f64a419e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 -- 172.21.15.90:0/349407425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f64a4101e70 msgr2=0x7f64a4197200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f64a4101e70 0x7f64a4197200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 -- 172.21.15.90:0/349407425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 msgr2=0x7f64a419e210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 0x7f64a419e210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 -- 172.21.15.90:0/349407425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f649c009d00 con 0x7f64a4100c80 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a9be6700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 0x7f64a419e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64a93e5700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f64a000d980 tx=0x7f64a000dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64a000de60 con 0x7f64a4100c80 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f64a00044e0 con 0x7f64a4100c80 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64a419edf0 con 0x7f64a4100c80 2024-04-06T14:04:09.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64a0003be0 con 0x7f64a4100c80 2024-04-06T14:04:09.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.760+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64a419f270 con 0x7f64a4100c80 2024-04-06T14:04:09.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.761+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f64a4108050 con 0x7f64a4100c80 2024-04-06T14:04:09.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.762+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f64a0009bb0 con 0x7f64a4100c80 2024-04-06T14:04:09.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.762+0000 7f64967fc700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 0x7f64900a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:09.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.763+0000 7f64a8be4700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 0x7f64900a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:09.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.763+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f64a00cc670 con 0x7f64a4100c80 2024-04-06T14:04:09.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.763+0000 7f64a8be4700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 0x7f64900a3670 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f6498005950 tx=0x7f64980058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:09.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.767+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f64a0095e50 con 0x7f64a4100c80 2024-04-06T14:04:09.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:09 smithi090 ceph-mon[43420]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:09.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:09 smithi090 ceph-mon[43420]: mgrmap e17: x(active, since 5m), standbys: z, y 2024-04-06T14:04:09.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2890994672' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:09.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3889399998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:09.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:09 smithi134 ceph-mon[47549]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:09.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:09 smithi134 ceph-mon[47549]: mgrmap e17: x(active, since 5m), standbys: z, y 2024-04-06T14:04:09.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2890994672' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:09.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3889399998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:09.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.962+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f64a402ce60 con 0x7f64900a11b0 2024-04-06T14:04:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.971+0000 7f64967fc700 1 -- 172.21.15.90:0/349407425 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+328352 (secure 0 0 0) 0x7f64a402ce60 con 0x7f64900a11b0 2024-04-06T14:04:09.975 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 msgr2=0x7f64900a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 0x7f64900a3670 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f6498005950 tx=0x7f64980058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 msgr2=0x7f64a4196cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f64a000d980 tx=0x7f64a000dd40 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 shutdown_connections 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f64900a11b0 0x7f64900a3670 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f64a4102810 0x7f64a419e210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f64a4100c80 0x7f64a4196cc0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 --2- 172.21.15.90:0/349407425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f64a4101e70 0x7f64a4197200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.974+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 >> 172.21.15.90:0/349407425 conn(0x7f64a40fc210 msgr2=0x7f64a4105190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:09.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.975+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 shutdown_connections 2024-04-06T14:04:09.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:09.975+0000 7f64ab649700 1 -- 172.21.15.90:0/349407425 wait complete. 2024-04-06T14:04:09.979 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:04:09.986 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":223,"stamp":"2024-04-06T14:04:08.159973+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593886,"num_objects":40,"num_object_clones":0,"num_object_copies":120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40,"num_whiteouts":0,"num_read":138,"num_read_kb":121,"num_write":263,"num_write_kb":2961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":224,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":359232,"kb_used_data":10412,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124582592,"statfs":{"total":1151940427776,"available":1151572574208,"internally_reserved":0,"allocated":10661888,"data_stored":6789005,"data_compressed":107160,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":3518,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":82,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.147102"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489854+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:04:01.489854+0000","last_peered":"2024-04-06T14:04:01.489854+0000","last_clean":"2024-04-06T14:04:01.489854+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:04:01.489854+0000","last_undegraded":"2024-04-06T14:04:01.489854+0000","last_fullsized":"2024-04-06T14:04:01.489854+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574988+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:04:02.574988+0000","last_peered":"2024-04-06T14:04:02.574988+0000","last_clean":"2024-04-06T14:04:02.574988+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:04:02.574988+0000","last_undegraded":"2024-04-06T14:04:02.574988+0000","last_fullsized":"2024-04-06T14:04:02.574988+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574769+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:04:02.574769+0000","last_peered":"2024-04-06T14:04:02.574769+0000","last_clean":"2024-04-06T14:04:02.574769+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:04:02.574769+0000","last_undegraded":"2024-04-06T14:04:02.574769+0000","last_fullsized":"2024-04-06T14:04:02.574769+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470214+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:04:01.470214+0000","last_peered":"2024-04-06T14:04:01.470214+0000","last_clean":"2024-04-06T14:04:01.470214+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:04:01.470214+0000","last_undegraded":"2024-04-06T14:04:01.470214+0000","last_fullsized":"2024-04-06T14:04:01.470214+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575085+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:04:02.575085+0000","last_peered":"2024-04-06T14:04:02.575085+0000","last_clean":"2024-04-06T14:04:02.575085+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:04:02.575085+0000","last_undegraded":"2024-04-06T14:04:02.575085+0000","last_fullsized":"2024-04-06T14:04:02.575085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436129+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:04:02.436129+0000","last_peered":"2024-04-06T14:04:02.436129+0000","last_clean":"2024-04-06T14:04:02.436129+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:04:02.436129+0000","last_undegraded":"2024-04-06T14:04:02.436129+0000","last_fullsized":"2024-04-06T14:04:02.436129+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225702+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:04:02.225702+0000","last_peered":"2024-04-06T14:04:02.225702+0000","last_clean":"2024-04-06T14:04:02.225702+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:04:02.225702+0000","last_undegraded":"2024-04-06T14:04:02.225702+0000","last_fullsized":"2024-04-06T14:04:02.225702+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575535+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:04:02.575535+0000","last_peered":"2024-04-06T14:04:02.575535+0000","last_clean":"2024-04-06T14:04:02.575535+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:04:02.575535+0000","last_undegraded":"2024-04-06T14:04:02.575535+0000","last_fullsized":"2024-04-06T14:04:02.575535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575974+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:04:02.575974+0000","last_peered":"2024-04-06T14:04:02.575974+0000","last_clean":"2024-04-06T14:04:02.575974+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:04:02.575974+0000","last_undegraded":"2024-04-06T14:04:02.575974+0000","last_fullsized":"2024-04-06T14:04:02.575974+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436412+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:04:02.436412+0000","last_peered":"2024-04-06T14:04:02.436412+0000","last_clean":"2024-04-06T14:04:02.436412+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:04:02.436412+0000","last_undegraded":"2024-04-06T14:04:02.436412+0000","last_fullsized":"2024-04-06T14:04:02.436412+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225803+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:04:02.225803+0000","last_peered":"2024-04-06T14:04:02.225803+0000","last_clean":"2024-04-06T14:04:02.225803+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:04:02.225803+0000","last_undegraded":"2024-04-06T14:04:02.225803+0000","last_fullsized":"2024-04-06T14:04:02.225803+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470202+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:04:01.470202+0000","last_peered":"2024-04-06T14:04:01.470202+0000","last_clean":"2024-04-06T14:04:01.470202+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:04:01.470202+0000","last_undegraded":"2024-04-06T14:04:01.470202+0000","last_fullsized":"2024-04-06T14:04:01.470202+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575159+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:04:02.575159+0000","last_peered":"2024-04-06T14:04:02.575159+0000","last_clean":"2024-04-06T14:04:02.575159+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:04:02.575159+0000","last_undegraded":"2024-04-06T14:04:02.575159+0000","last_fullsized":"2024-04-06T14:04:02.575159+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574684+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:04:02.574684+0000","last_peered":"2024-04-06T14:04:02.574684+0000","last_clean":"2024-04-06T14:04:02.574684+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:04:02.574684+0000","last_undegraded":"2024-04-06T14:04:02.574684+0000","last_fullsized":"2024-04-06T14:04:02.574684+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574459+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:04:02.574459+0000","last_peered":"2024-04-06T14:04:02.574459+0000","last_clean":"2024-04-06T14:04:02.574459+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:04:02.574459+0000","last_undegraded":"2024-04-06T14:04:02.574459+0000","last_fullsized":"2024-04-06T14:04:02.574459+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575094+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:04:02.575094+0000","last_peered":"2024-04-06T14:04:02.575094+0000","last_clean":"2024-04-06T14:04:02.575094+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:04:02.575094+0000","last_undegraded":"2024-04-06T14:04:02.575094+0000","last_fullsized":"2024-04-06T14:04:02.575094+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489865+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:04:01.489865+0000","last_peered":"2024-04-06T14:04:01.489865+0000","last_clean":"2024-04-06T14:04:01.489865+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:04:01.489865+0000","last_undegraded":"2024-04-06T14:04:01.489865+0000","last_fullsized":"2024-04-06T14:04:01.489865+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437817+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:04:02.437817+0000","last_peered":"2024-04-06T14:04:02.437817+0000","last_clean":"2024-04-06T14:04:02.437817+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:04:02.437817+0000","last_undegraded":"2024-04-06T14:04:02.437817+0000","last_fullsized":"2024-04-06T14:04:02.437817+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672247+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:04:01.672247+0000","last_peered":"2024-04-06T14:04:01.672247+0000","last_clean":"2024-04-06T14:04:01.672247+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:04:01.672247+0000","last_undegraded":"2024-04-06T14:04:01.672247+0000","last_fullsized":"2024-04-06T14:04:01.672247+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":21,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513680+0000","last_change":"2024-04-06T14:04:01.028407+0000","last_active":"2024-04-06T14:04:05.513680+0000","last_peered":"2024-04-06T14:04:05.513680+0000","last_clean":"2024-04-06T14:04:05.513680+0000","last_became_active":"2024-04-06T14:04:01.028036+0000","last_became_peered":"2024-04-06T14:04:01.028036+0000","last_unstale":"2024-04-06T14:04:05.513680+0000","last_undegraded":"2024-04-06T14:04:05.513680+0000","last_fullsized":"2024-04-06T14:04:05.513680+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:08:53.486282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575627+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:04:02.575627+0000","last_peered":"2024-04-06T14:04:02.575627+0000","last_clean":"2024-04-06T14:04:02.575627+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:04:02.575627+0000","last_undegraded":"2024-04-06T14:04:02.575627+0000","last_fullsized":"2024-04-06T14:04:02.575627+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575635+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:04:02.575635+0000","last_peered":"2024-04-06T14:04:02.575635+0000","last_clean":"2024-04-06T14:04:02.575635+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:04:02.575635+0000","last_undegraded":"2024-04-06T14:04:02.575635+0000","last_fullsized":"2024-04-06T14:04:02.575635+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575409+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:04:02.575409+0000","last_peered":"2024-04-06T14:04:02.575409+0000","last_clean":"2024-04-06T14:04:02.575409+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:04:02.575409+0000","last_undegraded":"2024-04-06T14:04:02.575409+0000","last_fullsized":"2024-04-06T14:04:02.575409+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575420+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:04:02.575420+0000","last_peered":"2024-04-06T14:04:02.575420+0000","last_clean":"2024-04-06T14:04:02.575420+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:04:02.575420+0000","last_undegraded":"2024-04-06T14:04:02.575420+0000","last_fullsized":"2024-04-06T14:04:02.575420+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575221+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:04:02.575221+0000","last_peered":"2024-04-06T14:04:02.575221+0000","last_clean":"2024-04-06T14:04:02.575221+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:04:02.575221+0000","last_undegraded":"2024-04-06T14:04:02.575221+0000","last_fullsized":"2024-04-06T14:04:02.575221+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575477+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:04:02.575477+0000","last_peered":"2024-04-06T14:04:02.575477+0000","last_clean":"2024-04-06T14:04:02.575477+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:04:02.575477+0000","last_undegraded":"2024-04-06T14:04:02.575477+0000","last_fullsized":"2024-04-06T14:04:02.575477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574918+0000","last_change":"2024-04-06T14:04:01.457311+0000","last_active":"2024-04-06T14:04:02.574918+0000","last_peered":"2024-04-06T14:04:02.574918+0000","last_clean":"2024-04-06T14:04:02.574918+0000","last_became_active":"2024-04-06T14:04:01.456954+0000","last_became_peered":"2024-04-06T14:04:01.456954+0000","last_unstale":"2024-04-06T14:04:02.574918+0000","last_undegraded":"2024-04-06T14:04:02.574918+0000","last_fullsized":"2024-04-06T14:04:02.574918+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:22:59.524995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574766+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:04:02.574766+0000","last_peered":"2024-04-06T14:04:02.574766+0000","last_clean":"2024-04-06T14:04:02.574766+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:04:02.574766+0000","last_undegraded":"2024-04-06T14:04:02.574766+0000","last_fullsized":"2024-04-06T14:04:02.574766+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575517+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:04:02.575517+0000","last_peered":"2024-04-06T14:04:02.575517+0000","last_clean":"2024-04-06T14:04:02.575517+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:04:02.575517+0000","last_undegraded":"2024-04-06T14:04:02.575517+0000","last_fullsized":"2024-04-06T14:04:02.575517+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436039+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:04:02.436039+0000","last_peered":"2024-04-06T14:04:02.436039+0000","last_clean":"2024-04-06T14:04:02.436039+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:04:02.436039+0000","last_undegraded":"2024-04-06T14:04:02.436039+0000","last_fullsized":"2024-04-06T14:04:02.436039+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574780+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:04:02.574780+0000","last_peered":"2024-04-06T14:04:02.574780+0000","last_clean":"2024-04-06T14:04:02.574780+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:04:02.574780+0000","last_undegraded":"2024-04-06T14:04:02.574780+0000","last_fullsized":"2024-04-06T14:04:02.574780+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.439215+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:04:01.439215+0000","last_peered":"2024-04-06T14:04:01.439215+0000","last_clean":"2024-04-06T14:04:01.439215+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:04:01.439215+0000","last_undegraded":"2024-04-06T14:04:01.439215+0000","last_fullsized":"2024-04-06T14:04:01.439215+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575621+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:04:02.575621+0000","last_peered":"2024-04-06T14:04:02.575621+0000","last_clean":"2024-04-06T14:04:02.575621+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:04:02.575621+0000","last_undegraded":"2024-04-06T14:04:02.575621+0000","last_fullsized":"2024-04-06T14:04:02.575621+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574562+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:04:02.574562+0000","last_peered":"2024-04-06T14:04:02.574562+0000","last_clean":"2024-04-06T14:04:02.574562+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:04:02.574562+0000","last_undegraded":"2024-04-06T14:04:02.574562+0000","last_fullsized":"2024-04-06T14:04:02.574562+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437700+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:04:02.437700+0000","last_peered":"2024-04-06T14:04:02.437700+0000","last_clean":"2024-04-06T14:04:02.437700+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:04:02.437700+0000","last_undegraded":"2024-04-06T14:04:02.437700+0000","last_fullsized":"2024-04-06T14:04:02.437700+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436356+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:04:02.436356+0000","last_peered":"2024-04-06T14:04:02.436356+0000","last_clean":"2024-04-06T14:04:02.436356+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:04:02.436356+0000","last_undegraded":"2024-04-06T14:04:02.436356+0000","last_fullsized":"2024-04-06T14:04:02.436356+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575670+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:04:02.575670+0000","last_peered":"2024-04-06T14:04:02.575670+0000","last_clean":"2024-04-06T14:04:02.575670+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:04:02.575670+0000","last_undegraded":"2024-04-06T14:04:02.575670+0000","last_fullsized":"2024-04-06T14:04:02.575670+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507363+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:04:04.507363+0000","last_peered":"2024-04-06T14:04:04.507363+0000","last_clean":"2024-04-06T14:04:04.507363+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:04:04.507363+0000","last_undegraded":"2024-04-06T14:04:04.507363+0000","last_fullsized":"2024-04-06T14:04:04.507363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575323+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:04:02.575323+0000","last_peered":"2024-04-06T14:04:02.575323+0000","last_clean":"2024-04-06T14:04:02.575323+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:04:02.575323+0000","last_undegraded":"2024-04-06T14:04:02.575323+0000","last_fullsized":"2024-04-06T14:04:02.575323+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"80'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575059+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:04:02.575059+0000","last_peered":"2024-04-06T14:04:02.575059+0000","last_clean":"2024-04-06T14:04:02.575059+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:04:02.575059+0000","last_undegraded":"2024-04-06T14:04:02.575059+0000","last_fullsized":"2024-04-06T14:04:02.575059+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575352+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:04:02.575352+0000","last_peered":"2024-04-06T14:04:02.575352+0000","last_clean":"2024-04-06T14:04:02.575352+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:04:02.575352+0000","last_undegraded":"2024-04-06T14:04:02.575352+0000","last_fullsized":"2024-04-06T14:04:02.575352+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"80'6","reported_seq":18,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436258+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:04:02.436258+0000","last_peered":"2024-04-06T14:04:02.436258+0000","last_clean":"2024-04-06T14:04:02.436258+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:04:02.436258+0000","last_undegraded":"2024-04-06T14:04:02.436258+0000","last_fullsized":"2024-04-06T14:04:02.436258+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575168+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:04:02.575168+0000","last_peered":"2024-04-06T14:04:02.575168+0000","last_clean":"2024-04-06T14:04:02.575168+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:04:02.575168+0000","last_undegraded":"2024-04-06T14:04:02.575168+0000","last_fullsized":"2024-04-06T14:04:02.575168+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":19,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513503+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:04:05.513503+0000","last_peered":"2024-04-06T14:04:05.513503+0000","last_clean":"2024-04-06T14:04:05.513503+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:04:05.513503+0000","last_undegraded":"2024-04-06T14:04:05.513503+0000","last_fullsized":"2024-04-06T14:04:05.513503+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574857+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:04:02.574857+0000","last_peered":"2024-04-06T14:04:02.574857+0000","last_clean":"2024-04-06T14:04:02.574857+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:04:02.574857+0000","last_undegraded":"2024-04-06T14:04:02.574857+0000","last_fullsized":"2024-04-06T14:04:02.574857+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574877+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:04:02.574877+0000","last_peered":"2024-04-06T14:04:02.574877+0000","last_clean":"2024-04-06T14:04:02.574877+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:04:02.574877+0000","last_undegraded":"2024-04-06T14:04:02.574877+0000","last_fullsized":"2024-04-06T14:04:02.574877+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225762+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:04:02.225762+0000","last_peered":"2024-04-06T14:04:02.225762+0000","last_clean":"2024-04-06T14:04:02.225762+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:04:02.225762+0000","last_undegraded":"2024-04-06T14:04:02.225762+0000","last_fullsized":"2024-04-06T14:04:02.225762+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512240+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:04:05.512240+0000","last_peered":"2024-04-06T14:04:05.512240+0000","last_clean":"2024-04-06T14:04:05.512240+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:04:05.512240+0000","last_undegraded":"2024-04-06T14:04:05.512240+0000","last_fullsized":"2024-04-06T14:04:05.512240+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225612+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:04:02.225612+0000","last_peered":"2024-04-06T14:04:02.225612+0000","last_clean":"2024-04-06T14:04:02.225612+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:04:02.225612+0000","last_undegraded":"2024-04-06T14:04:02.225612+0000","last_fullsized":"2024-04-06T14:04:02.225612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"80'8","reported_seq":20,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574985+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:04:02.574985+0000","last_peered":"2024-04-06T14:04:02.574985+0000","last_clean":"2024-04-06T14:04:02.574985+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:04:02.574985+0000","last_undegraded":"2024-04-06T14:04:02.574985+0000","last_fullsized":"2024-04-06T14:04:02.574985+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574613+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:04:02.574613+0000","last_peered":"2024-04-06T14:04:02.574613+0000","last_clean":"2024-04-06T14:04:02.574613+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:04:02.574613+0000","last_undegraded":"2024-04-06T14:04:02.574613+0000","last_fullsized":"2024-04-06T14:04:02.574613+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575394+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:04:02.575394+0000","last_peered":"2024-04-06T14:04:02.575394+0000","last_clean":"2024-04-06T14:04:02.575394+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:04:02.575394+0000","last_undegraded":"2024-04-06T14:04:02.575394+0000","last_fullsized":"2024-04-06T14:04:02.575394+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489844+0000","last_change":"2024-04-06T14:04:01.455152+0000","last_active":"2024-04-06T14:04:01.489844+0000","last_peered":"2024-04-06T14:04:01.489844+0000","last_clean":"2024-04-06T14:04:01.489844+0000","last_became_active":"2024-04-06T14:04:01.454827+0000","last_became_peered":"2024-04-06T14:04:01.454827+0000","last_unstale":"2024-04-06T14:04:01.489844+0000","last_undegraded":"2024-04-06T14:04:01.489844+0000","last_fullsized":"2024-04-06T14:04:01.489844+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:55:26.359159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.438338+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:04:01.438338+0000","last_peered":"2024-04-06T14:04:01.438338+0000","last_clean":"2024-04-06T14:04:01.438338+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:04:01.438338+0000","last_undegraded":"2024-04-06T14:04:01.438338+0000","last_fullsized":"2024-04-06T14:04:01.438338+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436512+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:04:02.436512+0000","last_peered":"2024-04-06T14:04:02.436512+0000","last_clean":"2024-04-06T14:04:02.436512+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:04:02.436512+0000","last_undegraded":"2024-04-06T14:04:02.436512+0000","last_fullsized":"2024-04-06T14:04:02.436512+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576581+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:04:02.576581+0000","last_peered":"2024-04-06T14:04:02.576581+0000","last_clean":"2024-04-06T14:04:02.576581+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:04:02.576581+0000","last_undegraded":"2024-04-06T14:04:02.576581+0000","last_fullsized":"2024-04-06T14:04:02.576581+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436536+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:04:02.436536+0000","last_peered":"2024-04-06T14:04:02.436536+0000","last_clean":"2024-04-06T14:04:02.436536+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:04:02.436536+0000","last_undegraded":"2024-04-06T14:04:02.436536+0000","last_fullsized":"2024-04-06T14:04:02.436536+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575047+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:04:02.575047+0000","last_peered":"2024-04-06T14:04:02.575047+0000","last_clean":"2024-04-06T14:04:02.575047+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:04:02.575047+0000","last_undegraded":"2024-04-06T14:04:02.575047+0000","last_fullsized":"2024-04-06T14:04:02.575047+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574547+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:04:02.574547+0000","last_peered":"2024-04-06T14:04:02.574547+0000","last_clean":"2024-04-06T14:04:02.574547+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:04:02.574547+0000","last_undegraded":"2024-04-06T14:04:02.574547+0000","last_fullsized":"2024-04-06T14:04:02.574547+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575189+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:04:02.575189+0000","last_peered":"2024-04-06T14:04:02.575189+0000","last_clean":"2024-04-06T14:04:02.575189+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:04:02.575189+0000","last_undegraded":"2024-04-06T14:04:02.575189+0000","last_fullsized":"2024-04-06T14:04:02.575189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.508080+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:04:04.508080+0000","last_peered":"2024-04-06T14:04:04.508080+0000","last_clean":"2024-04-06T14:04:04.508080+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:04:04.508080+0000","last_undegraded":"2024-04-06T14:04:04.508080+0000","last_fullsized":"2024-04-06T14:04:04.508080+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":333,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.451452+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:04:01.451452+0000","last_peered":"2024-04-06T14:04:01.451452+0000","last_clean":"2024-04-06T14:04:01.451452+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:04:01.451452+0000","last_undegraded":"2024-04-06T14:04:01.451452+0000","last_fullsized":"2024-04-06T14:04:01.451452+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574458+0000","last_change":"2024-04-06T14:04:01.457132+0000","last_active":"2024-04-06T14:04:02.574458+0000","last_peered":"2024-04-06T14:04:02.574458+0000","last_clean":"2024-04-06T14:04:02.574458+0000","last_became_active":"2024-04-06T14:04:01.456782+0000","last_became_peered":"2024-04-06T14:04:01.456782+0000","last_unstale":"2024-04-06T14:04:02.574458+0000","last_undegraded":"2024-04-06T14:04:02.574458+0000","last_fullsized":"2024-04-06T14:04:02.574458+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:34:37.759113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512264+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:04:05.512264+0000","last_peered":"2024-04-06T14:04:05.512264+0000","last_clean":"2024-04-06T14:04:05.512264+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:04:05.512264+0000","last_undegraded":"2024-04-06T14:04:05.512264+0000","last_fullsized":"2024-04-06T14:04:05.512264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575260+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:04:02.575260+0000","last_peered":"2024-04-06T14:04:02.575260+0000","last_clean":"2024-04-06T14:04:02.575260+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:04:02.575260+0000","last_undegraded":"2024-04-06T14:04:02.575260+0000","last_fullsized":"2024-04-06T14:04:02.575260+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574508+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:04:02.574508+0000","last_peered":"2024-04-06T14:04:02.574508+0000","last_clean":"2024-04-06T14:04:02.574508+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:04:02.574508+0000","last_undegraded":"2024-04-06T14:04:02.574508+0000","last_fullsized":"2024-04-06T14:04:02.574508+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513917+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:04:05.513917+0000","last_peered":"2024-04-06T14:04:05.513917+0000","last_clean":"2024-04-06T14:04:05.513917+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:04:05.513917+0000","last_undegraded":"2024-04-06T14:04:05.513917+0000","last_fullsized":"2024-04-06T14:04:05.513917+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575544+0000","last_change":"2024-04-06T14:04:01.461874+0000","last_active":"2024-04-06T14:04:02.575544+0000","last_peered":"2024-04-06T14:04:02.575544+0000","last_clean":"2024-04-06T14:04:02.575544+0000","last_became_active":"2024-04-06T14:04:01.461652+0000","last_became_peered":"2024-04-06T14:04:01.461652+0000","last_unstale":"2024-04-06T14:04:02.575544+0000","last_undegraded":"2024-04-06T14:04:02.575544+0000","last_fullsized":"2024-04-06T14:04:02.575544+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:01:58.862215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507264+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:04:04.507264+0000","last_peered":"2024-04-06T14:04:04.507264+0000","last_clean":"2024-04-06T14:04:04.507264+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:04:04.507264+0000","last_undegraded":"2024-04-06T14:04:04.507264+0000","last_fullsized":"2024-04-06T14:04:04.507264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575075+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:04:02.575075+0000","last_peered":"2024-04-06T14:04:02.575075+0000","last_clean":"2024-04-06T14:04:02.575075+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:04:02.575075+0000","last_undegraded":"2024-04-06T14:04:02.575075+0000","last_fullsized":"2024-04-06T14:04:02.575075+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507554+0000","last_change":"2024-04-06T14:04:01.035529+0000","last_active":"2024-04-06T14:04:04.507554+0000","last_peered":"2024-04-06T14:04:04.507554+0000","last_clean":"2024-04-06T14:04:04.507554+0000","last_became_active":"2024-04-06T14:04:01.035205+0000","last_became_peered":"2024-04-06T14:04:01.035205+0000","last_unstale":"2024-04-06T14:04:04.507554+0000","last_undegraded":"2024-04-06T14:04:04.507554+0000","last_fullsized":"2024-04-06T14:04:04.507554+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:59:17.605780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574775+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:04:02.574775+0000","last_peered":"2024-04-06T14:04:02.574775+0000","last_clean":"2024-04-06T14:04:02.574775+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:04:02.574775+0000","last_undegraded":"2024-04-06T14:04:02.574775+0000","last_fullsized":"2024-04-06T14:04:02.574775+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574687+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:04:02.574687+0000","last_peered":"2024-04-06T14:04:02.574687+0000","last_clean":"2024-04-06T14:04:02.574687+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:04:02.574687+0000","last_undegraded":"2024-04-06T14:04:02.574687+0000","last_fullsized":"2024-04-06T14:04:02.574687+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489773+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:04:01.489773+0000","last_peered":"2024-04-06T14:04:01.489773+0000","last_clean":"2024-04-06T14:04:01.489773+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:04:01.489773+0000","last_undegraded":"2024-04-06T14:04:01.489773+0000","last_fullsized":"2024-04-06T14:04:01.489773+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672174+0000","last_change":"2024-04-06T14:04:01.032612+0000","last_active":"2024-04-06T14:04:01.672174+0000","last_peered":"2024-04-06T14:04:01.672174+0000","last_clean":"2024-04-06T14:04:01.672174+0000","last_became_active":"2024-04-06T14:04:01.032115+0000","last_became_peered":"2024-04-06T14:04:01.032115+0000","last_unstale":"2024-04-06T14:04:01.672174+0000","last_undegraded":"2024-04-06T14:04:01.672174+0000","last_fullsized":"2024-04-06T14:04:01.672174+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:40.607887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575432+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:04:02.575432+0000","last_peered":"2024-04-06T14:04:02.575432+0000","last_clean":"2024-04-06T14:04:02.575432+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:04:02.575432+0000","last_undegraded":"2024-04-06T14:04:02.575432+0000","last_fullsized":"2024-04-06T14:04:02.575432+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575163+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:04:02.575163+0000","last_peered":"2024-04-06T14:04:02.575163+0000","last_clean":"2024-04-06T14:04:02.575163+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:04:02.575163+0000","last_undegraded":"2024-04-06T14:04:02.575163+0000","last_fullsized":"2024-04-06T14:04:02.575163+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225742+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:04:02.225742+0000","last_peered":"2024-04-06T14:04:02.225742+0000","last_clean":"2024-04-06T14:04:02.225742+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:04:02.225742+0000","last_undegraded":"2024-04-06T14:04:02.225742+0000","last_fullsized":"2024-04-06T14:04:02.225742+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513714+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:04:05.513714+0000","last_peered":"2024-04-06T14:04:05.513714+0000","last_clean":"2024-04-06T14:04:05.513714+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:04:05.513714+0000","last_undegraded":"2024-04-06T14:04:05.513714+0000","last_fullsized":"2024-04-06T14:04:05.513714+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225824+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:04:02.225824+0000","last_peered":"2024-04-06T14:04:02.225824+0000","last_clean":"2024-04-06T14:04:02.225824+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:04:02.225824+0000","last_undegraded":"2024-04-06T14:04:02.225824+0000","last_fullsized":"2024-04-06T14:04:02.225824+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"80'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574856+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:04:02.574856+0000","last_peered":"2024-04-06T14:04:02.574856+0000","last_clean":"2024-04-06T14:04:02.574856+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:04:02.574856+0000","last_undegraded":"2024-04-06T14:04:02.574856+0000","last_fullsized":"2024-04-06T14:04:02.574856+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225783+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:04:02.225783+0000","last_peered":"2024-04-06T14:04:02.225783+0000","last_clean":"2024-04-06T14:04:02.225783+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:04:02.225783+0000","last_undegraded":"2024-04-06T14:04:02.225783+0000","last_fullsized":"2024-04-06T14:04:02.225783+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":14,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.467592+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:04:01.467592+0000","last_peered":"2024-04-06T14:04:01.467592+0000","last_clean":"2024-04-06T14:04:01.467592+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:04:01.467592+0000","last_undegraded":"2024-04-06T14:04:01.467592+0000","last_fullsized":"2024-04-06T14:04:01.467592+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225872+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:04:02.225872+0000","last_peered":"2024-04-06T14:04:02.225872+0000","last_clean":"2024-04-06T14:04:02.225872+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:04:02.225872+0000","last_undegraded":"2024-04-06T14:04:02.225872+0000","last_fullsized":"2024-04-06T14:04:02.225872+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489791+0000","last_change":"2024-04-06T14:04:01.454256+0000","last_active":"2024-04-06T14:04:01.489791+0000","last_peered":"2024-04-06T14:04:01.489791+0000","last_clean":"2024-04-06T14:04:01.489791+0000","last_became_active":"2024-04-06T14:04:01.452355+0000","last_became_peered":"2024-04-06T14:04:01.452355+0000","last_unstale":"2024-04-06T14:04:01.489791+0000","last_undegraded":"2024-04-06T14:04:01.489791+0000","last_fullsized":"2024-04-06T14:04:01.489791+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:34:42.131742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672108+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:04:01.672108+0000","last_peered":"2024-04-06T14:04:01.672108+0000","last_clean":"2024-04-06T14:04:01.672108+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:04:01.672108+0000","last_undegraded":"2024-04-06T14:04:01.672108+0000","last_fullsized":"2024-04-06T14:04:01.672108+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672134+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:04:01.672134+0000","last_peered":"2024-04-06T14:04:01.672134+0000","last_clean":"2024-04-06T14:04:01.672134+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:04:01.672134+0000","last_undegraded":"2024-04-06T14:04:01.672134+0000","last_fullsized":"2024-04-06T14:04:01.672134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574869+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:04:02.574869+0000","last_peered":"2024-04-06T14:04:02.574869+0000","last_clean":"2024-04-06T14:04:02.574869+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:04:02.574869+0000","last_undegraded":"2024-04-06T14:04:02.574869+0000","last_fullsized":"2024-04-06T14:04:02.574869+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575055+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:04:02.575055+0000","last_peered":"2024-04-06T14:04:02.575055+0000","last_clean":"2024-04-06T14:04:02.575055+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:04:02.575055+0000","last_undegraded":"2024-04-06T14:04:02.575055+0000","last_fullsized":"2024-04-06T14:04:02.575055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489833+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:04:01.489833+0000","last_peered":"2024-04-06T14:04:01.489833+0000","last_clean":"2024-04-06T14:04:01.489833+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:04:01.489833+0000","last_undegraded":"2024-04-06T14:04:01.489833+0000","last_fullsized":"2024-04-06T14:04:01.489833+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672268+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:04:01.672268+0000","last_peered":"2024-04-06T14:04:01.672268+0000","last_clean":"2024-04-06T14:04:01.672268+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:04:01.672268+0000","last_undegraded":"2024-04-06T14:04:01.672268+0000","last_fullsized":"2024-04-06T14:04:01.672268+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575443+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:04:02.575443+0000","last_peered":"2024-04-06T14:04:02.575443+0000","last_clean":"2024-04-06T14:04:02.575443+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:04:02.575443+0000","last_undegraded":"2024-04-06T14:04:02.575443+0000","last_fullsized":"2024-04-06T14:04:02.575443+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574980+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:04:02.574980+0000","last_peered":"2024-04-06T14:04:02.574980+0000","last_clean":"2024-04-06T14:04:02.574980+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:04:02.574980+0000","last_undegraded":"2024-04-06T14:04:02.574980+0000","last_fullsized":"2024-04-06T14:04:02.574980+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574778+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:04:02.574778+0000","last_peered":"2024-04-06T14:04:02.574778+0000","last_clean":"2024-04-06T14:04:02.574778+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:04:02.574778+0000","last_undegraded":"2024-04-06T14:04:02.574778+0000","last_fullsized":"2024-04-06T14:04:02.574778+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575547+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:04:02.575547+0000","last_peered":"2024-04-06T14:04:02.575547+0000","last_clean":"2024-04-06T14:04:02.575547+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:04:02.575547+0000","last_undegraded":"2024-04-06T14:04:02.575547+0000","last_fullsized":"2024-04-06T14:04:02.575547+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575324+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:04:02.575324+0000","last_peered":"2024-04-06T14:04:02.575324+0000","last_clean":"2024-04-06T14:04:02.575324+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:04:02.575324+0000","last_undegraded":"2024-04-06T14:04:02.575324+0000","last_fullsized":"2024-04-06T14:04:02.575324+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489822+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:04:01.489822+0000","last_peered":"2024-04-06T14:04:01.489822+0000","last_clean":"2024-04-06T14:04:01.489822+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:04:01.489822+0000","last_undegraded":"2024-04-06T14:04:01.489822+0000","last_fullsized":"2024-04-06T14:04:01.489822+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":376832,"data_stored":4132576,"data_compressed":246766,"data_compressed_allocated":315392,"data_compressed_original":4096000,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3518,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":82,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":159744,"data_stored":11253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70,"ondisk_log_size":70,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514507,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":868352,"data_stored":417921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73099999999999998}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76300000000000001}]}]},{"osd":10,"up_from":68,"seq":292057776141,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":933888,"data_stored":1009134,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[1,2,3,4,5,6,7,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41999999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40600000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69199999999999995}]},{"osd":9,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":9,"up_from":63,"seq":270582939665,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":418123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.45700000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.75700000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.33700000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.621},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.72699999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.46999999999999997}]},{"osd":6,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.495}]},{"osd":7,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.746}]},{"osd":8,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.67200000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68600000000000005}]}]},{"osd":8,"up_from":56,"seq":240518168596,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27872,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717280,"statfs":{"total":95995035648,"available":95966494720,"internally_reserved":0,"allocated":868352,"data_stored":417612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.48799999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56499999999999995}]},{"osd":2,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.38800000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.57899999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.628},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53700000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]}]},{"osd":7,"up_from":49,"seq":210453397528,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32016,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713136,"statfs":{"total":95995035648,"available":95962251264,"internally_reserved":0,"allocated":925696,"data_stored":1009072,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.629}]},{"osd":4,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.505}]},{"osd":5,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.84499999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.745}]},{"osd":6,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66800000000000004}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.78600000000000003}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.70199999999999996}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.754}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83299999999999996}]}]},{"osd":6,"up_from":45,"seq":193273528347,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":844,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":864256,"data_stored":417339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.315},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.39600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51000000000000001,"15min":0.502},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46899999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.504}]},{"osd":5,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83599999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.63600000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.62}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.57099999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.53900000000000003}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":876544,"data_stored":418903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46800000000000003,"15min":0.46999999999999997},"min":{"1min":0.36099999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.47599999999999998,"15min":0.46899999999999997},"min":{"1min":0.32200000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Apr 6 14:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.51800000000000002,"15min":0.496},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75600000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52400000000000002,"15min":0.50700000000000001},"min":{"1min":0.39900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70399999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Apr 6 14:03:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.48599999999999999,"15min":0.47999999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.44800000000000001,"15min":0.442},"min":{"1min":0.30399999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77200000000000002,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.42299999999999999,"15min":0.41099999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.35399999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79200000000000004,"5min":0.877,"15min":0.877},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56899999999999995,"15min":0.55700000000000005},"min":{"1min":0.48499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.432},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.38100000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.74399999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.69199999999999995}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27868,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717284,"statfs":{"total":95995035648,"available":95966498816,"internally_reserved":0,"allocated":868352,"data_stored":417429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51700000000000002},"min":{"1min":0.32700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.67200000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63800000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.375}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.502,"15min":0.5},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.498}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.443,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46700000000000003,"15min":0.46899999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70799999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:03:39 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.56499999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.48799999999999999}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27896,"kb_used_data":880,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717256,"statfs":{"total":95995035648,"available":95966470144,"internally_reserved":0,"allocated":901120,"data_stored":420205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.47299999999999998,"15min":0.45300000000000001},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45600000000000002,"15min":0.439},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53500000000000003,"15min":0.51700000000000002},"min":{"1min":0.438,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:03:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71199999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51900000000000002,"15min":0.51100000000000001},"min":{"1min":0.432,"5min":0.248,"15min":0.248},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.55200000000000005,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.42999999999999999},"min":{"1min":0.32700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.42399999999999999,"15min":0.41499999999999998},"min":{"1min":0.31900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60599999999999998,"15min":0.59999999999999998},"min":{"1min":0.44,"5min":0.372,"15min":0.372},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.75700000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.74299999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.79400000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71199999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81799999999999995}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36064,"kb_used_data":852,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93709088,"statfs":{"total":95995035648,"available":95958106112,"internally_reserved":0,"allocated":872448,"data_stored":417443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.33400000000000002},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55100000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.372},{"interface":"front","average":{"1min":0.379,"5min":0.33800000000000002,"15min":0.33100000000000002},"min":{"1min":0.25700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.36499999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.49299999999999999,"15min":0.48499999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.67400000000000004}]},{"osd":2,"last update":"Sat Apr 6 14:03:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.53100000000000003,"15min":0.52400000000000002},"min":{"1min":0.41299999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.442,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Apr 6 14:03:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54500000000000004,"15min":0.54300000000000004},"min":{"1min":0.41299999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.57999999999999996,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.46700000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:03:03 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:03:22 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.626},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.68999999999999995}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.438}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70799999999999996}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71599999999999997}]}]},{"osd":4,"up_from":33,"seq":141733920803,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":876544,"data_stored":417935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.38,"15min":0.371},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.497}]},{"osd":2,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.52300000000000002,"15min":0.51400000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.53900000000000003,"15min":0.53000000000000003},"min":{"1min":0.36399999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.747}]},{"osd":3,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.50600000000000001,"15min":0.502},"min":{"1min":0.34899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.47299999999999998,"15min":0.46899999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.55600000000000005,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.48299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.81799999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.621,"5min":0.52400000000000002,"15min":0.50800000000000001},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:03:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.621},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.65100000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.75,"5min":0.75,"15min":0.75},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.694,"5min":1.694,"15min":1.694},"last":0.68200000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:03:53 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.66700000000000004}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81100000000000005}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83599999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32028,"kb_used_data":904,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713124,"statfs":{"total":95995035648,"available":95962238976,"internally_reserved":0,"allocated":925696,"data_stored":1007889,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.49299999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.48999999999999999,"15min":0.47699999999999998},"min":{"1min":0.36599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.52400000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51100000000000001,"15min":0.497},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.748},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.499,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.83799999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.38300000000000001,"15min":0.37},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.317,"5min":0.223,"15min":0.223},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.52900000000000003,"15min":0.51600000000000001},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57199999999999995,"15min":0.56200000000000006},"min":{"1min":0.45400000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83199999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.627,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.876,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.76200000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.873}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.81999999999999995}]},{"osd":10,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":2866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:04:09.992 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2024-04-06T14:04:09.992 INFO:tasks.cephadm:Setting up client nodes... 2024-04-06T14:04:09.993 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-06T14:04:09.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:09 smithi105 ceph-mon[46969]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "fs subvolume getpath", "vol_name": "cephfs", "group_name": "qa", "sub_name": "sv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:09.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:09 smithi105 ceph-mon[46969]: mgrmap e17: x(active, since 5m), standbys: z, y 2024-04-06T14:04:09.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2890994672' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:09.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3889399998' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:10.245 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:10.762 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:10 smithi090 ceph-mon[43420]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.762 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:10 smithi090 ceph-mon[43420]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.762 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:10 smithi090 ceph-mon[43420]: pgmap v224: 97 pgs: 97 active+clean; 617 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8.7 KiB/s wr, 17 op/s 2024-04-06T14:04:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:10 smithi134 ceph-mon[47549]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:10 smithi134 ceph-mon[47549]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:10 smithi134 ceph-mon[47549]: pgmap v224: 97 pgs: 97 active+clean; 617 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8.7 KiB/s wr, 17 op/s 2024-04-06T14:04:10.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:10 smithi105 ceph-mon[46969]: from='client.14787 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:10 smithi105 ceph-mon[46969]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:10.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:10 smithi105 ceph-mon[46969]: pgmap v224: 97 pgs: 97 active+clean; 617 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8.7 KiB/s wr, 17 op/s 2024-04-06T14:04:11.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.347+0000 7f9d0032e700 1 -- 172.21.15.90:0/3535429154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf80ffce0 msgr2=0x7f9cf8100100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.347+0000 7f9d0032e700 1 --2- 172.21.15.90:0/3535429154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8100100 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f9ce800b0d0 tx=0x7f9ce800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 -- 172.21.15.90:0/3535429154 shutdown_connections 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 --2- 172.21.15.90:0/3535429154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8101880 0x7f9cf8108130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 --2- 172.21.15.90:0/3535429154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8100100 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 --2- 172.21.15.90:0/3535429154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf8100ee0 0x7f9cf8101340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 -- 172.21.15.90:0/3535429154 >> 172.21.15.90:0/3535429154 conn(0x7f9cf80fb280 msgr2=0x7f9cf80fd6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 -- 172.21.15.90:0/3535429154 shutdown_connections 2024-04-06T14:04:11.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 -- 172.21.15.90:0/3535429154 wait complete. 2024-04-06T14:04:11.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.348+0000 7f9d0032e700 1 Processor -- start 2024-04-06T14:04:11.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 -- start start 2024-04-06T14:04:11.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8195da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:11.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8100ee0 0x7f9cf81962e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cf819d8e0 con 0x7f9cf8101880 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cf819da20 con 0x7f9cf80ffce0 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9d0032e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cf819dbb0 con 0x7f9cf8100ee0 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34214/0 (socket says 172.21.15.90:34214) 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 -- 172.21.15.90:0/966003699 learned_addr learned my addr 172.21.15.90:0/966003699 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfd8c9700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8100ee0 0x7f9cf81962e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe0ca700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8195da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 -- 172.21.15.90:0/966003699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8100ee0 msgr2=0x7f9cf81962e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8100ee0 0x7f9cf81962e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 -- 172.21.15.90:0/966003699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 msgr2=0x7f9cf8195da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:11.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8195da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 -- 172.21.15.90:0/966003699 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ce8009d00 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe0ca700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8195da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cfe8cb700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f9cf0007b80 tx=0x7f9cf0007f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.349+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9cf0011070 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.350+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9cf000c2e0 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.350+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9cf8068940 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.350+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9cf000a980 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.350+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9cf8068e40 con 0x7f9cf8101880 2024-04-06T14:04:11.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.350+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9cf804ea80 con 0x7f9cf8101880 2024-04-06T14:04:11.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.352+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9cf000ab00 con 0x7f9cf8101880 2024-04-06T14:04:11.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.352+0000 7f9cf77fe700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 0x7f9ce00a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:11.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.352+0000 7f9cfe0ca700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 0x7f9ce00a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:11.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.352+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f9cf00ce200 con 0x7f9cf8101880 2024-04-06T14:04:11.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.353+0000 7f9cfe0ca700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 0x7f9ce00a36f0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f9ce8007bd0 tx=0x7f9ce800b040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:11.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.355+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9cf00979e0 con 0x7f9cf8101880 2024-04-06T14:04:11.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.628+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) v1 -- 0x7f9cf810b560 con 0x7f9cf8101880 2024-04-06T14:04:11.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.632+0000 7f9cf77fe700 1 -- 172.21.15.90:0/966003699 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v31) v1 ==== 170+0+59 (secure 0 0 0) 0x7f9cf000adb0 con 0x7f9cf8101880 2024-04-06T14:04:11.632 INFO:teuthology.orchestra.run.smithi090.stdout:[client.0] 2024-04-06T14:04:11.632 INFO:teuthology.orchestra.run.smithi090.stdout: key = AQBbVhFmZV+AJRAAOgegJwYjPfZIGaOuth6EHQ== 2024-04-06T14:04:11.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 msgr2=0x7f9ce00a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:11.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 0x7f9ce00a36f0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f9ce8007bd0 tx=0x7f9ce800b040 comp rx=0 tx=0).stop 2024-04-06T14:04:11.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 msgr2=0x7f9cf819d310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:11.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f9cf0007b80 tx=0x7f9cf0007f40 comp rx=0 tx=0).stop 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 shutdown_connections 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ce00a1230 0x7f9ce00a36f0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9cf8101880 0x7f9cf819d310 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9cf80ffce0 0x7f9cf8195da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 --2- 172.21.15.90:0/966003699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9cf8100ee0 0x7f9cf81962e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.634+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 >> 172.21.15.90:0/966003699 conn(0x7f9cf80fb280 msgr2=0x7f9cf8104300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.635+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 shutdown_connections 2024-04-06T14:04:11.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:11.635+0000 7f9d0032e700 1 -- 172.21.15.90:0/966003699 wait complete. 2024-04-06T14:04:11.957 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-06T14:04:11.957 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-06T14:04:11.976 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T14:04:11.976 DEBUG:teuthology.orchestra.run.smithi090:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-06T14:04:11.976 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-06T14:04:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:11 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-06T14:04:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:11 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-06T14:04:12.017 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-06T14:04:12.017 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-06T14:04:12.017 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-06T14:04:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:11 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-06T14:04:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:11 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/966003699' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-06T14:04:12.276 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:12.688 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:12 smithi090 ceph-mon[43420]: pgmap v225: 97 pgs: 97 active+clean; 618 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 7.2 KiB/s wr, 17 op/s 2024-04-06T14:04:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:12 smithi105 ceph-mon[46969]: pgmap v225: 97 pgs: 97 active+clean; 618 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 7.2 KiB/s wr, 17 op/s 2024-04-06T14:04:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:12 smithi134 ceph-mon[47549]: pgmap v225: 97 pgs: 97 active+clean; 618 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.8 KiB/s rd, 7.2 KiB/s wr, 17 op/s 2024-04-06T14:04:13.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 -- 172.21.15.90:0/2614489420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 msgr2=0x7f5b30107220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:13.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 --2- 172.21.15.90:0/2614489420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30107220 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f5b24009990 tx=0x7f5b24009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 -- 172.21.15.90:0/2614489420 shutdown_connections 2024-04-06T14:04:13.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 --2- 172.21.15.90:0/2614489420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b301017a0 0x7f5b3010dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 --2- 172.21.15.90:0/2614489420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30107220 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 --2- 172.21.15.90:0/2614489420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b30100e00 0x7f5b30101260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 -- 172.21.15.90:0/2614489420 >> 172.21.15.90:0/2614489420 conn(0x7f5b30077fa0 msgr2=0x7f5b300783b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 -- 172.21.15.90:0/2614489420 shutdown_connections 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.313+0000 7f5b37041700 1 -- 172.21.15.90:0/2614489420 wait complete. 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 Processor -- start 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 -- start start 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b30100e00 0x7f5b300730b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b301017a0 0x7f5b300735f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b301a56d0 con 0x7f5b30106e40 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b301a5840 con 0x7f5b301017a0 2024-04-06T14:04:13.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b37041700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b301a5a00 con 0x7f5b30100e00 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b355de700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b355de700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59180/0 (socket says 172.21.15.90:59180) 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b355de700 1 -- 172.21.15.90:0/4286658774 learned_addr learned my addr 172.21.15.90:0/4286658774 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.314+0000 7f5b2ffff700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b301017a0 0x7f5b300735f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b34ddd700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b30100e00 0x7f5b300730b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 -- 172.21.15.90:0/4286658774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b30100e00 msgr2=0x7f5b300730b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b30100e00 0x7f5b300730b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 -- 172.21.15.90:0/4286658774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b301017a0 msgr2=0x7f5b300735f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b301017a0 0x7f5b300735f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 -- 172.21.15.90:0/4286658774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b240095f0 con 0x7f5b30106e40 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b355de700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f5b2800b9e0 tx=0x7f5b2800bcf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b2800d8e0 con 0x7f5b30106e40 2024-04-06T14:04:13.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b28015420 con 0x7f5b30106e40 2024-04-06T14:04:13.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b301a5ce0 con 0x7f5b30106e40 2024-04-06T14:04:13.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b28014630 con 0x7f5b30106e40 2024-04-06T14:04:13.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.315+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b301a61e0 con 0x7f5b30106e40 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.316+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b30109360 con 0x7f5b30106e40 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.317+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f5b28015590 con 0x7f5b30106e40 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.318+0000 7f5b2dffb700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 0x7f5b200a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.318+0000 7f5b34ddd700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 0x7f5b200a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.318+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f5b28011070 con 0x7f5b30106e40 2024-04-06T14:04:13.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.318+0000 7f5b34ddd700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 0x7f5b200a36f0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f5b24003730 tx=0x7f5b240036c0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:13.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.322+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5b2804fab0 con 0x7f5b30106e40 2024-04-06T14:04:13.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.569+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7f5b30074590 con 0x7f5b30106e40 2024-04-06T14:04:13.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.573+0000 7f5b2dffb700 1 -- 172.21.15.90:0/4286658774 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v17) v1 ==== 74+0+260283 (secure 0 0 0) 0x7f5b280628d0 con 0x7f5b30106e40 2024-04-06T14:04:13.576 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 msgr2=0x7f5b200a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 0x7f5b200a36f0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f5b24003730 tx=0x7f5b240036c0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 msgr2=0x7f5b30073b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f5b2800b9e0 tx=0x7f5b2800bcf0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 shutdown_connections 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f5b200a1230 0x7f5b200a36f0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b30106e40 0x7f5b30073b30 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b30100e00 0x7f5b300730b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 --2- 172.21.15.90:0/4286658774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b301017a0 0x7f5b300735f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:13.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 >> 172.21.15.90:0/4286658774 conn(0x7f5b30077fa0 msgr2=0x7f5b300ffc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:13.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.576+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 shutdown_connections 2024-04-06T14:04:13.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:13.577+0000 7f5b37041700 1 -- 172.21.15.90:0/4286658774 wait complete. 2024-04-06T14:04:13.973 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:13 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4286658774' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-06T14:04:13.992 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":17,"flags":0,"active_gid":14152,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6800","nonce":1992032630},{"type":"v1","addr":"172.21.15.90:6801","nonce":1992032630}]},"active_addr":"172.21.15.90:6801/1992032630","active_change":"2024-04-06T13:59:04.100295+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24115,"name":"z","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]},{"gid":24116,"name":"y","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.90:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":319323044}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":582164689}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":2318576826}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":2298955235}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.90:0","nonce":177425578}]}]} 2024-04-06T14:04:13.998 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-06T14:04:13.998 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-06T14:04:13.998 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-06T14:04:13.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:13 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4286658774' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-06T14:04:14.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:13 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4286658774' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-06T14:04:14.222 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:14.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:14 smithi105 ceph-mon[46969]: pgmap v226: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 6.7 KiB/s wr, 16 op/s 2024-04-06T14:04:15.007 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:14 smithi090 ceph-mon[43420]: pgmap v226: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 6.7 KiB/s wr, 16 op/s 2024-04-06T14:04:15.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:14 smithi134 ceph-mon[47549]: pgmap v226: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 6.7 KiB/s wr, 16 op/s 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 -- 172.21.15.90:0/4059335424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30501015c0 msgr2=0x7f3050101a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 --2- 172.21.15.90:0/4059335424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30501015c0 0x7f3050101a40 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f3044009990 tx=0x7f3044009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 -- 172.21.15.90:0/4059335424 shutdown_connections 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 --2- 172.21.15.90:0/4059335424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3050101f80 0x7f305010ca90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 --2- 172.21.15.90:0/4059335424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30500ffe70 0x7f3050100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 --2- 172.21.15.90:0/4059335424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30501015c0 0x7f3050101a40 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 -- 172.21.15.90:0/4059335424 >> 172.21.15.90:0/4059335424 conn(0x7f30500fb5f0 msgr2=0x7f30500fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 -- 172.21.15.90:0/4059335424 shutdown_connections 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.379+0000 7f30551a0700 1 -- 172.21.15.90:0/4059335424 wait complete. 2024-04-06T14:04:15.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 Processor -- start 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 -- start start 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30501015c0 0x7f30500fbfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3050101f80 0x7f30500fc520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30501a5470 con 0x7f30500ffe70 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30501a55e0 con 0x7f3050101f80 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f30551a0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30501a57a0 con 0x7f30501015c0 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f304ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f304f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30501015c0 0x7f30500fbfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f304ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59204/0 (socket says 172.21.15.90:59204) 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f304ffff700 1 -- 172.21.15.90:0/899782930 learned_addr learned my addr 172.21.15.90:0/899782930 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.380+0000 7f305499f700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3050101f80 0x7f30500fc520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:15.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 -- 172.21.15.90:0/899782930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30501015c0 msgr2=0x7f30500fbfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30501015c0 0x7f30500fbfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 -- 172.21.15.90:0/899782930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3050101f80 msgr2=0x7f30500fc520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3050101f80 0x7f30500fc520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 -- 172.21.15.90:0/899782930 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30440095f0 con 0x7f30500ffe70 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304ffff700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f304000b530 tx=0x7f304000b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3040011830 con 0x7f30500ffe70 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3040011e50 con 0x7f30500ffe70 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f30501a5a80 con 0x7f30500ffe70 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f304000f2e0 con 0x7f30500ffe70 2024-04-06T14:04:15.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.381+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f30501a5f50 con 0x7f30500ffe70 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.382+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3050109180 con 0x7f30500ffe70 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.383+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f30400105a0 con 0x7f30500ffe70 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.384+0000 7f304d7fa700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 0x7f30380a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.384+0000 7f304f7fe700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 0x7f30380a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.384+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f30400cc6d0 con 0x7f30500ffe70 2024-04-06T14:04:15.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.384+0000 7f304f7fe700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 0x7f30380a3670 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f30440098e0 tx=0x7f3044005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:15.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.387+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3040095fc0 con 0x7f30500ffe70 2024-04-06T14:04:15.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.583+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f30500fcf80 con 0x7f30500ffe70 2024-04-06T14:04:15.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.584+0000 7f304d7fa700 1 -- 172.21.15.90:0/899782930 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7f30400995e0 con 0x7f30500ffe70 2024-04-06T14:04:15.584 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:15.584 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 msgr2=0x7f30380a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 0x7f30380a3670 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f30440098e0 tx=0x7f3044005960 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 msgr2=0x7f30500fbaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f304000b530 tx=0x7f304000b840 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 shutdown_connections 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30380a11b0 0x7f30380a3670 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3050101f80 0x7f30500fc520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30500ffe70 0x7f30500fbaa0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 --2- 172.21.15.90:0/899782930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30501015c0 0x7f30500fbfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 >> 172.21.15.90:0/899782930 conn(0x7f30500fb5f0 msgr2=0x7f3050068b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 shutdown_connections 2024-04-06T14:04:15.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:15.587+0000 7f30551a0700 1 -- 172.21.15.90:0/899782930 wait complete. 2024-04-06T14:04:15.898 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:15 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/899782930' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:15.974 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-06T14:04:15.974 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-06T14:04:15.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:15 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/899782930' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:16.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:15 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/899782930' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:16.195 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:16.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:16 smithi105 ceph-mon[46969]: pgmap v227: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.1 KiB/s wr, 14 op/s 2024-04-06T14:04:17.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:16 smithi090 ceph-mon[43420]: pgmap v227: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.1 KiB/s wr, 14 op/s 2024-04-06T14:04:17.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:16 smithi134 ceph-mon[47549]: pgmap v227: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.1 KiB/s wr, 14 op/s 2024-04-06T14:04:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- 172.21.15.90:0/3427899366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8100990 msgr2=0x7f00e810caa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 --2- 172.21.15.90:0/3427899366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8100990 0x7f00e810caa0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f00e000b0d0 tx=0x7f00e000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- 172.21.15.90:0/3427899366 shutdown_connections 2024-04-06T14:04:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 --2- 172.21.15.90:0/3427899366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8100990 0x7f00e810caa0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 --2- 172.21.15.90:0/3427899366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e8105e50 0x7f00e8106230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 --2- 172.21.15.90:0/3427899366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e80fffd0 0x7f00e8100450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- 172.21.15.90:0/3427899366 >> 172.21.15.90:0/3427899366 conn(0x7f00e8076a50 msgr2=0x7f00e8076e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- 172.21.15.90:0/3427899366 shutdown_connections 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- 172.21.15.90:0/3427899366 wait complete. 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 Processor -- start 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.254+0000 7f00ee999700 1 -- start start 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e80fffd0 0x7f00e8197220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e8100990 0x7f00e8197760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e819c120 con 0x7f00e8105e50 2024-04-06T14:04:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e819c290 con 0x7f00e80fffd0 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e819c450 con 0x7f00e8100990 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59224/0 (socket says 172.21.15.90:59224) 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 -- 172.21.15.90:0/3388427556 learned_addr learned my addr 172.21.15.90:0/3388427556 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00e7fff700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e80fffd0 0x7f00e8197220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00e77fe700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e8100990 0x7f00e8197760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 -- 172.21.15.90:0/3388427556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e8100990 msgr2=0x7f00e8197760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e8100990 0x7f00e8197760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 -- 172.21.15.90:0/3388427556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e80fffd0 msgr2=0x7f00e8197220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e80fffd0 0x7f00e8197220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 -- 172.21.15.90:0/3388427556 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00e0009d00 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ecf36700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f00e0009440 tx=0x7f00e0009470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00e000e040 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f00e001d940 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.255+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f00e819c6d0 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.256+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00e001b990 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.256+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f00e81a3f90 con 0x7f00e8105e50 2024-04-06T14:04:17.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.256+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f00e81080f0 con 0x7f00e8105e50 2024-04-06T14:04:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.258+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f00e001dab0 con 0x7f00e8105e50 2024-04-06T14:04:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.258+0000 7f00e57fa700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 0x7f00d00a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.258+0000 7f00e7fff700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 0x7f00d00a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.258+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f00e00ce1d0 con 0x7f00e8105e50 2024-04-06T14:04:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.259+0000 7f00e7fff700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 0x7f00d00a36f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f00d8005950 tx=0x7f00d800b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:17.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.262+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f00e0097a40 con 0x7f00e8105e50 2024-04-06T14:04:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.460+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f00e804ea80 con 0x7f00e8105e50 2024-04-06T14:04:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.461+0000 7f00e57fa700 1 -- 172.21.15.90:0/3388427556 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7f00e009b060 con 0x7f00e8105e50 2024-04-06T14:04:17.461 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:17.462 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:17.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 msgr2=0x7f00d00a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:17.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 0x7f00d00a36f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f00d8005950 tx=0x7f00d800b410 comp rx=0 tx=0).stop 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 msgr2=0x7f00e819bb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f00e0009440 tx=0x7f00e0009470 comp rx=0 tx=0).stop 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 shutdown_connections 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f00d00a1230 0x7f00d00a36f0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e8105e50 0x7f00e819bb10 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e80fffd0 0x7f00e8197220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 --2- 172.21.15.90:0/3388427556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e8100990 0x7f00e8197760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.463+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 >> 172.21.15.90:0/3388427556 conn(0x7f00e8076a50 msgr2=0x7f00e80fed50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.464+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 shutdown_connections 2024-04-06T14:04:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:17.464+0000 7f00ee999700 1 -- 172.21.15.90:0/3388427556 wait complete. 2024-04-06T14:04:17.773 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:17 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3388427556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:17.861 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-06T14:04:17.862 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-06T14:04:17.862 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-06T14:04:17.863 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-06T14:04:17.864 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-06T14:04:17.864 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-06T14:04:17.865 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-06T14:04:17.865 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-06T14:04:17.866 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.8 flush_pg_stats 2024-04-06T14:04:17.866 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.9 flush_pg_stats 2024-04-06T14:04:17.866 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.10 flush_pg_stats 2024-04-06T14:04:17.867 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph tell osd.11 flush_pg_stats 2024-04-06T14:04:17.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:17 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3388427556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:18.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:17 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3388427556' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:18.396 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:18.681 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:18.905 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:18 smithi090 ceph-mon[43420]: pgmap v228: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.0 KiB/s wr, 13 op/s 2024-04-06T14:04:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:18 smithi105 ceph-mon[46969]: pgmap v228: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.0 KiB/s wr, 13 op/s 2024-04-06T14:04:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:18 smithi134 ceph-mon[47549]: pgmap v228: 97 pgs: 97 active+clean; 623 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 6.0 KiB/s wr, 13 op/s 2024-04-06T14:04:19.389 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:19.439 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:20.664 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:20.664 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:20.901 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 -- 172.21.15.90:0/815198582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 msgr2=0x7f53fc100160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 --2- 172.21.15.90:0/815198582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc100160 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f53ec009990 tx=0x7f53ec009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 -- 172.21.15.90:0/815198582 shutdown_connections 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 --2- 172.21.15.90:0/815198582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f53fc1006a0 0x7f53fc10cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 --2- 172.21.15.90:0/815198582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f53fc105d10 0x7f53fc1060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 --2- 172.21.15.90:0/815198582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc100160 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 -- 172.21.15.90:0/815198582 >> 172.21.15.90:0/815198582 conn(0x7f53fc075d90 msgr2=0x7f53fc0761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:21.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 -- 172.21.15.90:0/815198582 shutdown_connections 2024-04-06T14:04:21.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.300+0000 7f5401a0d700 1 -- 172.21.15.90:0/815198582 wait complete. 2024-04-06T14:04:21.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.301+0000 7f5401a0d700 1 Processor -- start 2024-04-06T14:04:21.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.301+0000 7f5401a0d700 1 -- start start 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc19be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f53fc1006a0 0x7f53fc19c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f53fc105d10 0x7f53fc1974a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53fc197af0 con 0x7f53fc0ffce0 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53fc197c60 con 0x7f53fc1006a0 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53fc197e20 con 0x7f53fc105d10 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc19be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc19be10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59238/0 (socket says 172.21.15.90:59238) 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 -- 172.21.15.90:0/315219088 learned_addr learned my addr 172.21.15.90:0/315219088 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53fb7fe700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f53fc105d10 0x7f53fc1974a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53fa7fc700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f53fc1006a0 0x7f53fc19c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 -- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f53fc105d10 msgr2=0x7f53fc1974a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f53fc105d10 0x7f53fc1974a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 -- 172.21.15.90:0/315219088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f53fc1006a0 msgr2=0x7f53fc19c350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f53fc1006a0 0x7f53fc19c350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53ec0095f0 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f53faffd700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc19be10 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f53f000d950 tx=0x7f53f000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53f0009940 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f53f0010460 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f53fc198100 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53f000f620 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.302+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53fc1a3f60 con 0x7f53fc0ffce0 2024-04-06T14:04:21.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.303+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f53fc10fec0 con 0x7f53fc0ffce0 2024-04-06T14:04:21.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.304+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f53f00108d0 con 0x7f53fc0ffce0 2024-04-06T14:04:21.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f5400a0b700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f53e40a11b0 0x7f53e40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f53fa7fc700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f53e40a11b0 0x7f53e40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f53f00ccc00 con 0x7f53fc0ffce0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f5400a0b700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] conn(0x7f53e40aa990 0x7f53e40acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f53e40ad460 con 0x7f53e40aa990 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f53f00cd0a0 con 0x7f53fc0ffce0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f53fa7fc700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f53e40a11b0 0x7f53e40a3670 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f53ec005850 tx=0x7f53ec01a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.305+0000 7f53fb7fe700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] conn(0x7f53e40aa990 0x7f53e40acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.306+0000 7f53fb7fe700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] conn(0x7f53e40aa990 0x7f53e40acdb0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.8 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:21.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.307+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== osd.8 v2:172.21.15.134:6816/549202076 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f53e40ad460 con 0x7f53e40aa990 2024-04-06T14:04:21.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.323+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 --> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f53fc104b90 con 0x7f53e40aa990 2024-04-06T14:04:21.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.323+0000 7f5400a0b700 1 -- 172.21.15.90:0/315219088 <== osd.8 v2:172.21.15.134:6816/549202076 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f53fc104b90 con 0x7f53e40aa990 2024-04-06T14:04:21.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] conn(0x7f53e40aa990 msgr2=0x7f53e40acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] conn(0x7f53e40aa990 0x7f53e40acdb0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f53e40a11b0 msgr2=0x7f53e40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f53e40a11b0 0x7f53e40a3670 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f53ec005850 tx=0x7f53ec01a040 comp rx=0 tx=0).stop 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 msgr2=0x7f53fc19be10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 --2- 172.21.15.90:0/315219088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f53fc0ffce0 0x7f53fc19be10 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f53f000d950 tx=0x7f53f000dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f53fb7fe700 1 -- 172.21.15.90:0/315219088 reap_dead start 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 shutdown_connections 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 >> 172.21.15.90:0/315219088 conn(0x7f53fc075d90 msgr2=0x7f53fc0fddd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 shutdown_connections 2024-04-06T14:04:21.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:21.324+0000 7f5401a0d700 1 -- 172.21.15.90:0/315219088 wait complete. 2024-04-06T14:04:21.429 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:21 smithi090 ceph-mon[43420]: pgmap v229: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 6.5 KiB/s wr, 12 op/s 2024-04-06T14:04:21.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:21 smithi105 ceph-mon[46969]: pgmap v229: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 6.5 KiB/s wr, 12 op/s 2024-04-06T14:04:21.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:21 smithi134 ceph-mon[47549]: pgmap v229: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 8.9 KiB/s rd, 6.5 KiB/s wr, 12 op/s 2024-04-06T14:04:22.206 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:22.232 INFO:teuthology.orchestra.run.smithi090.stdout:240518168600 2024-04-06T14:04:22.232 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.8 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.350+0000 7f87f6378700 1 -- 172.21.15.90:0/4133704669 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f0106e40 msgr2=0x7f87f0107220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.350+0000 7f87f6378700 1 --2- 172.21.15.90:0/4133704669 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f0106e40 0x7f87f0107220 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f87e000b0a0 tx=0x7f87e000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 -- 172.21.15.90:0/4133704669 shutdown_connections 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 --2- 172.21.15.90:0/4133704669 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f87f01017a0 0x7f87f010dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 --2- 172.21.15.90:0/4133704669 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f0106e40 0x7f87f0107220 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 --2- 172.21.15.90:0/4133704669 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87f0100e00 0x7f87f0101260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 -- 172.21.15.90:0/4133704669 >> 172.21.15.90:0/4133704669 conn(0x7f87f0077fa0 msgr2=0x7f87f00783b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:22.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 -- 172.21.15.90:0/4133704669 shutdown_connections 2024-04-06T14:04:22.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 -- 172.21.15.90:0/4133704669 wait complete. 2024-04-06T14:04:22.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.351+0000 7f87f6378700 1 Processor -- start 2024-04-06T14:04:22.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 -- start start 2024-04-06T14:04:22.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f87f0100e00 0x7f87f00730b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 0x7f87f00735f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87f0106e40 0x7f87f0073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87f0075070 con 0x7f87f01017a0 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87f019b540 con 0x7f87f0106e40 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f6378700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87f019b700 con 0x7f87f0100e00 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 0x7f87f00735f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87e77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 0x7f87f00735f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59256/0 (socket says 172.21.15.90:59256) 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87e77fe700 1 -- 172.21.15.90:0/2125194754 learned_addr learned my addr 172.21.15.90:0/2125194754 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87f4915700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87f0106e40 0x7f87f0073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.352+0000 7f87effff700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f87f0100e00 0x7f87f00730b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 -- 172.21.15.90:0/2125194754 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f87f0100e00 msgr2=0x7f87f00730b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f87f0100e00 0x7f87f00730b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 -- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87f0106e40 msgr2=0x7f87f0073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87f0106e40 0x7f87f0073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87e0009d00 con 0x7f87f01017a0 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.353+0000 7f87e77fe700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 0x7f87f00735f0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f87d800c920 tx=0x7f87d800cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.354+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87d8004a30 con 0x7f87f01017a0 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.354+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f87d80054e0 con 0x7f87f01017a0 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.354+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f87d8007bd0 con 0x7f87f01017a0 2024-04-06T14:04:22.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.355+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f87f019b9e0 con 0x7f87f01017a0 2024-04-06T14:04:22.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.355+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f87f019be90 con 0x7f87f01017a0 2024-04-06T14:04:22.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.356+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f87f0111000 con 0x7f87f01017a0 2024-04-06T14:04:22.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.358+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f87d8007d30 con 0x7f87f01017a0 2024-04-06T14:04:22.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.358+0000 7f87edffb700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f87dc0a1230 0x7f87dc0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.358+0000 7f87effff700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f87dc0a1230 0x7f87dc0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.358+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f87d80ccb10 con 0x7f87f01017a0 2024-04-06T14:04:22.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87edffb700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] conn(0x7f87dc0aaa10 0x7f87dc0ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f87dc0ad4e0 con 0x7f87dc0aaa10 2024-04-06T14:04:22.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f87d80ccfb0 con 0x7f87f01017a0 2024-04-06T14:04:22.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87effff700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f87dc0a1230 0x7f87dc0a36f0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f87e0009fd0 tx=0x7f87e000b7b0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87f4915700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] conn(0x7f87dc0aaa10 0x7f87dc0ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.359+0000 7f87f4915700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] conn(0x7f87dc0aaa10 0x7f87dc0ace30 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.10 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.360+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== osd.10 v2:172.21.15.105:6824/394238659 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f87dc0ad4e0 con 0x7f87dc0aaa10 2024-04-06T14:04:22.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.376+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 --> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f87f0061430 con 0x7f87dc0aaa10 2024-04-06T14:04:22.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87edffb700 1 -- 172.21.15.90:0/2125194754 <== osd.10 v2:172.21.15.105:6824/394238659 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f87f0061430 con 0x7f87dc0aaa10 2024-04-06T14:04:22.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] conn(0x7f87dc0aaa10 msgr2=0x7f87dc0ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] conn(0x7f87dc0aaa10 0x7f87dc0ace30 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f87dc0a1230 msgr2=0x7f87dc0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f87dc0a1230 0x7f87dc0a36f0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f87e0009fd0 tx=0x7f87e000b7b0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 msgr2=0x7f87f00735f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 --2- 172.21.15.90:0/2125194754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87f01017a0 0x7f87f00735f0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f87d800c920 tx=0x7f87d800cc30 comp rx=0 tx=0).stop 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f4915700 1 -- 172.21.15.90:0/2125194754 reap_dead start 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 shutdown_connections 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 >> 172.21.15.90:0/2125194754 conn(0x7f87f0077fa0 msgr2=0x7f87f00ffcf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 shutdown_connections 2024-04-06T14:04:22.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.377+0000 7f87f6378700 1 -- 172.21.15.90:0/2125194754 wait complete. 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 -- 172.21.15.90:0/3964448154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc1008a0 msgr2=0x7fc7bc1048f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3964448154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc1048f0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fc7b4009990 tx=0x7fc7b4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 -- 172.21.15.90:0/3964448154 shutdown_connections 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3964448154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc10cba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3964448154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7bc0ffef0 0x7fc7bc1002d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.420+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3964448154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc1048f0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.421+0000 7fc7c4800700 1 -- 172.21.15.90:0/3964448154 >> 172.21.15.90:0/3964448154 conn(0x7fc7bc076d50 msgr2=0x7fc7bc077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.421+0000 7fc7c4800700 1 -- 172.21.15.90:0/3964448154 shutdown_connections 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.421+0000 7fc7c4800700 1 -- 172.21.15.90:0/3964448154 wait complete. 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.421+0000 7fc7c4800700 1 Processor -- start 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.421+0000 7fc7c4800700 1 -- start start 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7bc0ffef0 0x7fc7bc197240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc197780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc19bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7bc19c140 con 0x7fc7bc104fd0 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7bc19c2b0 con 0x7fc7bc0ffef0 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7bc19c470 con 0x7fc7bc1008a0 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc197780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc19bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc19bb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59274/0 (socket says 172.21.15.90:59274) 2024-04-06T14:04:22.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc197780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:43542/0 (socket says 172.21.15.90:43542) 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 -- 172.21.15.90:0/3043019045 learned_addr learned my addr 172.21.15.90:0/3043019045 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c259c700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7bc0ffef0 0x7fc7bc197240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 -- 172.21.15.90:0/3043019045 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc1008a0 msgr2=0x7fc7bc197780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7bc1008a0 0x7fc7bc197780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 -- 172.21.15.90:0/3043019045 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7bc0ffef0 msgr2=0x7fc7bc197240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7bc0ffef0 0x7fc7bc197240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7b40095f0 con 0x7fc7bc104fd0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c2d9d700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc19bb30 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fc7b8007ee0 tx=0x7fc7b800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7b8017070 con 0x7fc7bc104fd0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7b800f040 con 0x7fc7bc104fd0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7bc19c750 con 0x7fc7bc104fd0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7b8005580 con 0x7fc7bc104fd0 2024-04-06T14:04:22.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.422+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7bc1a3fb0 con 0x7fc7bc104fd0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.423+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fc7bc1013d0 con 0x7fc7bc104fd0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.425+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fc7b800db20 con 0x7fc7bc104fd0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.425+0000 7fc7af7fe700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc7a80a1230 0x7fc7a80a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.425+0000 7fc7c259c700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc7a80a1230 0x7fc7a80a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fc7b80cc680 con 0x7fc7bc104fd0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7af7fe700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] conn(0x7fc7a80aaa10 0x7fc7a80ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fc7a80ad4e0 con 0x7fc7a80aaa10 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc7b80cc9f0 con 0x7fc7bc104fd0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7c259c700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc7a80a1230 0x7fc7a80a36f0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fc7b0005950 tx=0x7fc7b000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7c1d9b700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] conn(0x7fc7a80aaa10 0x7fc7a80ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:22.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.426+0000 7fc7c1d9b700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] conn(0x7fc7a80aaa10 0x7fc7a80ace30 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:22.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.427+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== osd.6 v2:172.21.15.90:6818/1241504412 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7fc7a80ad4e0 con 0x7fc7a80aaa10 2024-04-06T14:04:22.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.442+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 --> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fc7bc111030 con 0x7fc7a80aaa10 2024-04-06T14:04:22.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7af7fe700 1 -- 172.21.15.90:0/3043019045 <== osd.6 v2:172.21.15.90:6818/1241504412 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7fc7bc111030 con 0x7fc7a80aaa10 2024-04-06T14:04:22.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] conn(0x7fc7a80aaa10 msgr2=0x7fc7a80ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] conn(0x7fc7a80aaa10 0x7fc7a80ace30 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc7a80a1230 msgr2=0x7fc7a80a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc7a80a1230 0x7fc7a80a36f0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fc7b0005950 tx=0x7fc7b000b410 comp rx=0 tx=0).stop 2024-04-06T14:04:22.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 msgr2=0x7fc7bc19bb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:22.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 --2- 172.21.15.90:0/3043019045 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7bc104fd0 0x7fc7bc19bb30 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fc7b8007ee0 tx=0x7fc7b800d3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:22.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c2d9d700 1 -- 172.21.15.90:0/3043019045 reap_dead start 2024-04-06T14:04:22.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 shutdown_connections 2024-04-06T14:04:22.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 >> 172.21.15.90:0/3043019045 conn(0x7fc7bc076d50 msgr2=0x7fc7bc0fdf30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:22.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 shutdown_connections 2024-04-06T14:04:22.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:22.443+0000 7fc7c4800700 1 -- 172.21.15.90:0/3043019045 wait complete. 2024-04-06T14:04:22.613 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:22.613 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:23.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 -- 172.21.15.90:0/3487156998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 msgr2=0x7f919c1060f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3487156998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c1060f0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f9184009990 tx=0x7f9184009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 -- 172.21.15.90:0/3487156998 shutdown_connections 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3487156998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f919c1006a0 0x7f919c10cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3487156998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c1060f0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3487156998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f919c0ffce0 0x7f919c100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 -- 172.21.15.90:0/3487156998 >> 172.21.15.90:0/3487156998 conn(0x7f919c075d90 msgr2=0x7f919c0761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 -- 172.21.15.90:0/3487156998 shutdown_connections 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 -- 172.21.15.90:0/3487156998 wait complete. 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.183+0000 7f91a12d6700 1 Processor -- start 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 -- start start 2024-04-06T14:04:23.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f919c0ffce0 0x7f919c1972a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f919c1006a0 0x7f919c1977e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c19bb90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c19c1a0 con 0x7f919c105d10 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c19c310 con 0x7f919c1006a0 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f91a12d6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f919c19c4d0 con 0x7f919c0ffce0 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c19bb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c19bb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59300/0 (socket says 172.21.15.90:59300) 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 -- 172.21.15.90:0/3617167797 learned_addr learned my addr 172.21.15.90:0/3617167797 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919a7fc700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f919c1006a0 0x7f919c1977e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919affd700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f919c0ffce0 0x7f919c1972a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 -- 172.21.15.90:0/3617167797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f919c0ffce0 msgr2=0x7f919c1972a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f919c0ffce0 0x7f919c1972a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 -- 172.21.15.90:0/3617167797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f919c1006a0 msgr2=0x7f919c1977e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f919c1006a0 0x7f919c1977e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91840095f0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.184+0000 7f919b7fe700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c19bb90 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f919400b970 tx=0x7f919400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f919400c9b0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9194014440 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91940075d0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f919c19c7b0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f919c1a4010 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.185+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f919c1013c0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f91940145b0 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91880a1230 0x7f91880a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f919affd700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91880a1230 0x7f91880a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f9194006e40 con 0x7f919c105d10 2024-04-06T14:04:23.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] conn(0x7f91880aaa10 0x7f91880ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f91880ad4e0 con 0x7f91880aaa10 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f91940cc960 con 0x7f919c105d10 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.187+0000 7f919a7fc700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] conn(0x7f91880aaa10 0x7f91880ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.188+0000 7f919affd700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91880a1230 0x7f91880a36f0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f9184006010 tx=0x7f918400b540 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.188+0000 7f919a7fc700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] conn(0x7f91880aaa10 0x7f91880ace30 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.188+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== osd.3 v2:172.21.15.90:6810/711321384 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f91880ad4e0 con 0x7f91880aaa10 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.204+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 --> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f919c111090 con 0x7f91880aaa10 2024-04-06T14:04:23.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.204+0000 7f9193fff700 1 -- 172.21.15.90:0/3617167797 <== osd.3 v2:172.21.15.90:6810/711321384 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f919c111090 con 0x7f91880aaa10 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] conn(0x7f91880aaa10 msgr2=0x7f91880ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] conn(0x7f91880aaa10 0x7f91880ace30 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91880a1230 msgr2=0x7f91880a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f91880a1230 0x7f91880a36f0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f9184006010 tx=0x7f918400b540 comp rx=0 tx=0).stop 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 msgr2=0x7f919c19bb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 --2- 172.21.15.90:0/3617167797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f919c105d10 0x7f919c19bb90 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f919400b970 tx=0x7f919400bd30 comp rx=0 tx=0).stop 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f919b7fe700 1 -- 172.21.15.90:0/3617167797 reap_dead start 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 shutdown_connections 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 >> 172.21.15.90:0/3617167797 conn(0x7f919c075d90 msgr2=0x7f919c0fea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 shutdown_connections 2024-04-06T14:04:23.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.205+0000 7f91a12d6700 1 -- 172.21.15.90:0/3617167797 wait complete. 2024-04-06T14:04:23.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:23 smithi105 ceph-mon[46969]: pgmap v230: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-06T14:04:23.535 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:23 smithi090 ceph-mon[43420]: pgmap v230: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-06T14:04:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:23 smithi134 ceph-mon[47549]: pgmap v230: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-06T14:04:23.675 INFO:teuthology.orchestra.run.smithi090.stdout:292057776145 2024-04-06T14:04:23.675 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.10 2024-04-06T14:04:24.002 INFO:teuthology.orchestra.run.smithi090.stdout:111669149738 2024-04-06T14:04:24.002 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- 172.21.15.90:0/3045677824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 msgr2=0x7f9bd81060f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/3045677824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd81060f0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f9bc8009990 tx=0x7f9bc8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- 172.21.15.90:0/3045677824 shutdown_connections 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/3045677824 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bd81006a0 0x7f9bd810cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/3045677824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd81060f0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/3045677824 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bd80ffce0 0x7f9bd8100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- 172.21.15.90:0/3045677824 >> 172.21.15.90:0/3045677824 conn(0x7f9bd8075d90 msgr2=0x7f9bd80761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- 172.21.15.90:0/3045677824 shutdown_connections 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- 172.21.15.90:0/3045677824 wait complete. 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 Processor -- start 2024-04-06T14:04:24.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.955+0000 7f9bde9c5700 1 -- start start 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bd80ffce0 0x7f9bd8196120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bd81006a0 0x7f9bd8196660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd819aa10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819b020 con 0x7f9bd8105d10 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819b190 con 0x7f9bd80ffce0 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bde9c5700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819b350 con 0x7f9bd81006a0 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd819aa10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd819aa10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45284/0 (socket says 172.21.15.90:45284) 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 -- 172.21.15.90:0/1038021480 learned_addr learned my addr 172.21.15.90:0/1038021480 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bd7fff700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bd80ffce0 0x7f9bd8196120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bd77fe700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bd81006a0 0x7f9bd8196660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 -- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bd81006a0 msgr2=0x7f9bd8196660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bd81006a0 0x7f9bd8196660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 -- 172.21.15.90:0/1038021480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bd80ffce0 msgr2=0x7f9bd8196120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bd80ffce0 0x7f9bd8196120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bc80095f0 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bdcf62700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd819aa10 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0007ee0 tx=0x7f9bd000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.956+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd0017070 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.957+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bd819b630 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.957+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bd000f040 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.957+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd0005740 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.957+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bd81a2e90 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.957+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9bd81013c0 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.958+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9bd000dc60 con 0x7f9bd8105d10 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd57fa700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9bc40a11b0 0x7f9bc40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:24.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd7fff700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9bc40a11b0 0x7f9bc40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f9bd00cc710 con 0x7f9bd8105d10 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd57fa700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] conn(0x7f9bc40aa990 0x7f9bc40acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f9bc40ad460 con 0x7f9bc40aa990 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9bd00ccbb0 con 0x7f9bd8105d10 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.959+0000 7f9bd7fff700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9bc40a11b0 0x7f9bc40a3670 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9bc800b5c0 tx=0x7f9bc8005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.960+0000 7f9bd77fe700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] conn(0x7f9bc40aa990 0x7f9bc40acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.960+0000 7f9bd77fe700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] conn(0x7f9bc40aa990 0x7f9bc40acdb0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.961+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== osd.2 v2:172.21.15.134:6800/1139095317 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f9bc40ad460 con 0x7f9bc40aa990 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.976+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 --> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9bd810fe10 con 0x7f9bc40aa990 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.977+0000 7f9bd57fa700 1 -- 172.21.15.90:0/1038021480 <== osd.2 v2:172.21.15.134:6800/1139095317 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f9bd810fe10 con 0x7f9bc40aa990 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.977+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] conn(0x7f9bc40aa990 msgr2=0x7f9bc40acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.977+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] conn(0x7f9bc40aa990 0x7f9bc40acdb0 crc :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9bc40a11b0 msgr2=0x7f9bc40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9bc40a11b0 0x7f9bc40a3670 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f9bc800b5c0 tx=0x7f9bc8005960 comp rx=0 tx=0).stop 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 msgr2=0x7f9bd819aa10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 --2- 172.21.15.90:0/1038021480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bd8105d10 0x7f9bd819aa10 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0007ee0 tx=0x7f9bd000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bdcf62700 1 -- 172.21.15.90:0/1038021480 reap_dead start 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 shutdown_connections 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 >> 172.21.15.90:0/1038021480 conn(0x7f9bd8075d90 msgr2=0x7f9bd80fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 shutdown_connections 2024-04-06T14:04:24.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:23.978+0000 7f9bde9c5700 1 -- 172.21.15.90:0/1038021480 wait complete. 2024-04-06T14:04:25.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:25 smithi105 ceph-mon[46969]: pgmap v231: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2024-04-06T14:04:25.513 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:25 smithi090 ceph-mon[43420]: pgmap v231: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2024-04-06T14:04:25.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:25 smithi134 ceph-mon[47549]: pgmap v231: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 1.8 KiB/s wr, 7 op/s 2024-04-06T14:04:26.051 INFO:teuthology.orchestra.run.smithi090.stdout:77309411374 2024-04-06T14:04:26.052 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-06T14:04:26.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 -- 172.21.15.90:0/2578628911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f8069650 msgr2=0x7f51f8069a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 --2- 172.21.15.90:0/2578628911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f8069650 0x7f51f8069a30 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f51f4009990 tx=0x7f51f4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 -- 172.21.15.90:0/2578628911 shutdown_connections 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 --2- 172.21.15.90:0/2578628911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51f81060e0 0x7f51f810dcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 --2- 172.21.15.90:0/2578628911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f8069650 0x7f51f8069a30 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 --2- 172.21.15.90:0/2578628911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51f806a000 0x7f51f8105a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 -- 172.21.15.90:0/2578628911 >> 172.21.15.90:0/2578628911 conn(0x7f51f8076a50 msgr2=0x7f51f8076e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 -- 172.21.15.90:0/2578628911 shutdown_connections 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.114+0000 7f52005bd700 1 -- 172.21.15.90:0/2578628911 wait complete. 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 Processor -- start 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 -- start start 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51f8069650 0x7f51f81000a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51f806a000 0x7f51f81005e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 0x7f51f8100b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51f81a55a0 con 0x7f51f81060e0 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51f81a5710 con 0x7f51f8069650 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f52005bd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51f81a58d0 con 0x7f51f806a000 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f51feb5a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 0x7f51f8100b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f51feb5a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 0x7f51f8100b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45310/0 (socket says 172.21.15.90:45310) 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f51feb5a700 1 -- 172.21.15.90:0/1827007573 learned_addr learned my addr 172.21.15.90:0/1827007573 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f51fdb58700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51f806a000 0x7f51f81005e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.115+0000 7f51fe359700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51f8069650 0x7f51f81000a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 -- 172.21.15.90:0/1827007573 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51f806a000 msgr2=0x7f51f81005e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f51f806a000 0x7f51f81005e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 -- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51f8069650 msgr2=0x7f51f81000a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51f8069650 0x7f51f81000a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51f40095f0 con 0x7f51f81060e0 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51feb5a700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 0x7f51f8100b20 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f51f0007ee0 tx=0x7f51f000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51f0017070 con 0x7f51f81060e0 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f51f81a5bb0 con 0x7f51f81060e0 2024-04-06T14:04:26.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f51f000f040 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f51f0005580 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.116+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f51f81a60b0 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.117+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f51f81024e0 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.118+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f51f000db20 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.118+0000 7f51ef7fe700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51e40a11b0 0x7f51e40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51fe359700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51e40a11b0 0x7f51e40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f51f00cc6b0 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51ef7fe700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] conn(0x7f51e40aa990 0x7f51e40acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f51e40ad460 con 0x7f51e40aa990 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f51f00ccb50 con 0x7f51f81060e0 2024-04-06T14:04:26.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51fe359700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51e40a11b0 0x7f51e40a3670 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f51f400b5c0 tx=0x7f51f4005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51fdb58700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] conn(0x7f51e40aa990 0x7f51e40acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.119+0000 7f51fdb58700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] conn(0x7f51e40aa990 0x7f51e40acdb0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.120+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== osd.1 v2:172.21.15.105:6800/1321301965 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f51e40ad460 con 0x7f51e40aa990 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.136+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 --> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f51f8110fb0 con 0x7f51e40aa990 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f51ef7fe700 1 -- 172.21.15.90:0/1827007573 <== osd.1 v2:172.21.15.105:6800/1321301965 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f51f8110fb0 con 0x7f51e40aa990 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] conn(0x7f51e40aa990 msgr2=0x7f51e40acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] conn(0x7f51e40aa990 0x7f51e40acdb0 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51e40a11b0 msgr2=0x7f51e40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f51e40a11b0 0x7f51e40a3670 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f51f400b5c0 tx=0x7f51f4005960 comp rx=0 tx=0).stop 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 msgr2=0x7f51f8100b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 --2- 172.21.15.90:0/1827007573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f51f81060e0 0x7f51f8100b20 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f51f0007ee0 tx=0x7f51f000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f51feb5a700 1 -- 172.21.15.90:0/1827007573 reap_dead start 2024-04-06T14:04:26.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 shutdown_connections 2024-04-06T14:04:26.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 >> 172.21.15.90:0/1827007573 conn(0x7f51f8076a50 msgr2=0x7f51f81072b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:26.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 shutdown_connections 2024-04-06T14:04:26.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.137+0000 7f52005bd700 1 -- 172.21.15.90:0/1827007573 wait complete. 2024-04-06T14:04:26.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 -- 172.21.15.90:0/2425496228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b8103200 msgr2=0x7f28b81035e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 --2- 172.21.15.90:0/2425496228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b8103200 0x7f28b81035e0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f28a8009990 tx=0x7f28a8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 -- 172.21.15.90:0/2425496228 shutdown_connections 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 --2- 172.21.15.90:0/2425496228 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28b81082e0 0x7f28b810feb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 --2- 172.21.15.90:0/2425496228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b8103200 0x7f28b81035e0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 --2- 172.21.15.90:0/2425496228 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28b8103bb0 0x7f28b8107c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 -- 172.21.15.90:0/2425496228 >> 172.21.15.90:0/2425496228 conn(0x7f28b80fea70 msgr2=0x7f28b8100e90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 -- 172.21.15.90:0/2425496228 shutdown_connections 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.619+0000 7f28be9ed700 1 -- 172.21.15.90:0/2425496228 wait complete. 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 Processor -- start 2024-04-06T14:04:26.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 -- start start 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28b8103200 0x7f28b8199400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28b8103bb0 0x7f28b8199940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 0x7f28b819dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28b819e300 con 0x7f28b81082e0 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28b819e470 con 0x7f28b8103200 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28be9ed700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28b819e630 con 0x7f28b8103bb0 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28bcf8a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 0x7f28b819dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28bcf8a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 0x7f28b819dcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45320/0 (socket says 172.21.15.90:45320) 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.620+0000 7f28bcf8a700 1 -- 172.21.15.90:0/707740024 learned_addr learned my addr 172.21.15.90:0/707740024 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:26.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28b7fff700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28b8103200 0x7f28b8199400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28b77fe700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28b8103bb0 0x7f28b8199940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 -- 172.21.15.90:0/707740024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28b8103bb0 msgr2=0x7f28b8199940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28b8103bb0 0x7f28b8199940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 -- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28b8103200 msgr2=0x7f28b8199400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28b8103200 0x7f28b8199400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28a80095f0 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28bcf8a700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 0x7f28b819dcf0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f28b0007ee0 tx=0x7f28b000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28b000dd20 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28b819e910 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28b000f040 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28b00048a0 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.621+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28b81a6170 con 0x7f28b81082e0 2024-04-06T14:04:26.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.622+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f28b81046e0 con 0x7f28b81082e0 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.623+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f28b0003a00 con 0x7f28b81082e0 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b57fa700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28a40a11b0 0x7f28a40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b7fff700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28a40a11b0 0x7f28a40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f28b00ccbe0 con 0x7f28b81082e0 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b57fa700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] conn(0x7f28a40aa990 0x7f28a40acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f28a40ad460 con 0x7f28a40aa990 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f28b00cd080 con 0x7f28b81082e0 2024-04-06T14:04:26.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b7fff700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28a40a11b0 0x7f28a40a3670 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f28a8000c00 tx=0x7f28a8005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.624+0000 7f28b77fe700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] conn(0x7f28a40aa990 0x7f28a40acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:26.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.625+0000 7f28b77fe700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] conn(0x7f28a40aa990 0x7f28a40acdb0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:26.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.626+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== osd.4 v2:172.21.15.105:6808/4184734218 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f28a40ad460 con 0x7f28a40aa990 2024-04-06T14:04:26.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.641+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 --> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f28b8113130 con 0x7f28a40aa990 2024-04-06T14:04:26.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.641+0000 7f28b57fa700 1 -- 172.21.15.90:0/707740024 <== osd.4 v2:172.21.15.105:6808/4184734218 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f28b8113130 con 0x7f28a40aa990 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] conn(0x7f28a40aa990 msgr2=0x7f28a40acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] conn(0x7f28a40aa990 0x7f28a40acdb0 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28a40a11b0 msgr2=0x7f28a40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f28a40a11b0 0x7f28a40a3670 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f28a8000c00 tx=0x7f28a8005960 comp rx=0 tx=0).stop 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 msgr2=0x7f28b819dcf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 --2- 172.21.15.90:0/707740024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28b81082e0 0x7f28b819dcf0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f28b0007ee0 tx=0x7f28b000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28bcf8a700 1 -- 172.21.15.90:0/707740024 reap_dead start 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 shutdown_connections 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 >> 172.21.15.90:0/707740024 conn(0x7f28b80fea70 msgr2=0x7f28b80ffe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 shutdown_connections 2024-04-06T14:04:26.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:26.642+0000 7f28be9ed700 1 -- 172.21.15.90:0/707740024 wait complete. 2024-04-06T14:04:27.039 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:27.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 -- 172.21.15.90:0/2742660236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 msgr2=0x7f65c41060f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 --2- 172.21.15.90:0/2742660236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41060f0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f65b8009990 tx=0x7f65b8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 -- 172.21.15.90:0/2742660236 shutdown_connections 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 --2- 172.21.15.90:0/2742660236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65c41006a0 0x7f65c410cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 --2- 172.21.15.90:0/2742660236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41060f0 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 --2- 172.21.15.90:0/2742660236 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65c40ffce0 0x7f65c4100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 -- 172.21.15.90:0/2742660236 >> 172.21.15.90:0/2742660236 conn(0x7f65c4075d90 msgr2=0x7f65c40761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 -- 172.21.15.90:0/2742660236 shutdown_connections 2024-04-06T14:04:27.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 -- 172.21.15.90:0/2742660236 wait complete. 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.181+0000 7f65c9077700 1 Processor -- start 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 -- start start 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65c40ffce0 0x7f65c4197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65c41006a0 0x7f65c41984f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41a3030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65c4198d40 con 0x7f65c4105d10 2024-04-06T14:04:27.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65c41a3570 con 0x7f65c41006a0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c9077700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65c41a36b0 con 0x7f65c40ffce0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41a3030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41a3030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45340/0 (socket says 172.21.15.90:45340) 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 -- 172.21.15.90:0/1681641250 learned_addr learned my addr 172.21.15.90:0/1681641250 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c2d9d700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65c40ffce0 0x7f65c4197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c259c700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65c41006a0 0x7f65c41984f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 -- 172.21.15.90:0/1681641250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65c40ffce0 msgr2=0x7f65c4197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65c40ffce0 0x7f65c4197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 -- 172.21.15.90:0/1681641250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65c41006a0 msgr2=0x7f65c41984f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65c41006a0 0x7f65c41984f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.182+0000 7f65c359e700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65b80095f0 con 0x7f65c4105d10 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65c359e700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41a3030 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f65bc007f10 tx=0x7f65bc00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65bc017070 con 0x7f65c4105d10 2024-04-06T14:04:27.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65c41a3910 con 0x7f65c4105d10 2024-04-06T14:04:27.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65bc00f040 con 0x7f65c4105d10 2024-04-06T14:04:27.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65bc005740 con 0x7f65c4105d10 2024-04-06T14:04:27.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.183+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65c41a3e10 con 0x7f65c4105d10 2024-04-06T14:04:27.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.184+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f65c41013c0 con 0x7f65c4105d10 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.185+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f65bc0058a0 con 0x7f65c4105d10 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.185+0000 7f65abfff700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f65ac0a11b0 0x7f65ac0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.185+0000 7f65c2d9d700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f65ac0a11b0 0x7f65ac0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.185+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f65bc0cc6c0 con 0x7f65c4105d10 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65abfff700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] conn(0x7f65ac0aa990 0x7f65ac0acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f65ac0ad460 con 0x7f65ac0aa990 2024-04-06T14:04:27.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f65bc0ccb60 con 0x7f65c4105d10 2024-04-06T14:04:27.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65c259c700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] conn(0x7f65ac0aa990 0x7f65ac0acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65c2d9d700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f65ac0a11b0 0x7f65ac0a3670 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f65b80098e0 tx=0x7f65b8005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.186+0000 7f65c259c700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] conn(0x7f65ac0aa990 0x7f65ac0acdb0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.9 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.187+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== osd.9 v2:172.21.15.90:6826/154728526 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f65ac0ad460 con 0x7f65ac0aa990 2024-04-06T14:04:27.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.202+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 --> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f65c410fe50 con 0x7f65ac0aa990 2024-04-06T14:04:27.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65abfff700 1 -- 172.21.15.90:0/1681641250 <== osd.9 v2:172.21.15.90:6826/154728526 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f65c410fe50 con 0x7f65ac0aa990 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] conn(0x7f65ac0aa990 msgr2=0x7f65ac0acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] conn(0x7f65ac0aa990 0x7f65ac0acdb0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f65ac0a11b0 msgr2=0x7f65ac0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f65ac0a11b0 0x7f65ac0a3670 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f65b80098e0 tx=0x7f65b8005960 comp rx=0 tx=0).stop 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 msgr2=0x7f65c41a3030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 --2- 172.21.15.90:0/1681641250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65c4105d10 0x7f65c41a3030 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f65bc007f10 tx=0x7f65bc00d3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c359e700 1 -- 172.21.15.90:0/1681641250 reap_dead start 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 shutdown_connections 2024-04-06T14:04:27.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 >> 172.21.15.90:0/1681641250 conn(0x7f65c4075d90 msgr2=0x7f65c40feac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 shutdown_connections 2024-04-06T14:04:27.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.203+0000 7f65c9077700 1 -- 172.21.15.90:0/1681641250 wait complete. 2024-04-06T14:04:27.299 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:27 smithi090 ceph-mon[43420]: pgmap v232: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-06T14:04:27.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:27 smithi105 ceph-mon[46969]: pgmap v232: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 -- 172.21.15.90:0/4001705457 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 msgr2=0x7f7c4010cb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 --2- 172.21.15.90:0/4001705457 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c4010cb50 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f7c3000b0a0 tx=0x7f7c3000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 -- 172.21.15.90:0/4001705457 shutdown_connections 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 --2- 172.21.15.90:0/4001705457 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c4010cb50 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 --2- 172.21.15.90:0/4001705457 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c40105d10 0x7f7c401060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 --2- 172.21.15.90:0/4001705457 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c400ffce0 0x7f7c40100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 -- 172.21.15.90:0/4001705457 >> 172.21.15.90:0/4001705457 conn(0x7f7c40075d90 msgr2=0x7f7c400761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 -- 172.21.15.90:0/4001705457 shutdown_connections 2024-04-06T14:04:27.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.525+0000 7f7c476ee700 1 -- 172.21.15.90:0/4001705457 wait complete. 2024-04-06T14:04:27.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 Processor -- start 2024-04-06T14:04:27.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 -- start start 2024-04-06T14:04:27.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c400ffce0 0x7f7c40196130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c40196670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c40105d10 0x7f7c4019aa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c4019b030 con 0x7f7c401006a0 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c4019b1a0 con 0x7f7c40105d10 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c476ee700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c4019b360 con 0x7f7c400ffce0 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c44c89700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c40196670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c44c89700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c40196670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45358/0 (socket says 172.21.15.90:45358) 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c44c89700 1 -- 172.21.15.90:0/1931214271 learned_addr learned my addr 172.21.15.90:0/1931214271 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c4548a700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c400ffce0 0x7f7c40196130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.526+0000 7f7c45c8b700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c40105d10 0x7f7c4019aa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 -- 172.21.15.90:0/1931214271 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c400ffce0 msgr2=0x7f7c40196130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7c400ffce0 0x7f7c40196130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 -- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c40105d10 msgr2=0x7f7c4019aa20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7c40105d10 0x7f7c4019aa20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c30009d00 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c44c89700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c40196670 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f7c3400c9b0 tx=0x7f7c3400ccc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c34007b00 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c3400ce50 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c4019b640 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c34018660 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.527+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c401a2ea0 con 0x7f7c401006a0 2024-04-06T14:04:27.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.528+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f7c4010fe40 con 0x7f7c401006a0 2024-04-06T14:04:27.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.529+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f7c3400f450 con 0x7f7c401006a0 2024-04-06T14:04:27.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.529+0000 7f7c3e7fc700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c280a11b0 0x7f7c280a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c4548a700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c280a11b0 0x7f7c280a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f7c340ccc40 con 0x7f7c401006a0 2024-04-06T14:04:27.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c3e7fc700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] conn(0x7f7c280aa990 0x7f7c280acdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f7c280ad460 con 0x7f7c280aa990 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7c340cd0e0 con 0x7f7c401006a0 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c4548a700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c280a11b0 0x7f7c280a3670 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f7c38005950 tx=0x7f7c3800a300 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c45c8b700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] conn(0x7f7c280aa990 0x7f7c280acdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.530+0000 7f7c45c8b700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] conn(0x7f7c280aa990 0x7f7c280acdb0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.531+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== osd.7 v2:172.21.15.105:6816/4248271964 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f7c280ad460 con 0x7f7c280aa990 2024-04-06T14:04:27.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.547+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 --> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f7c400614b0 con 0x7f7c280aa990 2024-04-06T14:04:27.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.547+0000 7f7c3e7fc700 1 -- 172.21.15.90:0/1931214271 <== osd.7 v2:172.21.15.105:6816/4248271964 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f7c400614b0 con 0x7f7c280aa990 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] conn(0x7f7c280aa990 msgr2=0x7f7c280acdb0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] conn(0x7f7c280aa990 0x7f7c280acdb0 crc :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c280a11b0 msgr2=0x7f7c280a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7c280a11b0 0x7f7c280a3670 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f7c38005950 tx=0x7f7c3800a300 comp rx=0 tx=0).stop 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 msgr2=0x7f7c40196670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 --2- 172.21.15.90:0/1931214271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7c401006a0 0x7f7c40196670 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f7c3400c9b0 tx=0x7f7c3400ccc0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c45c8b700 1 -- 172.21.15.90:0/1931214271 reap_dead start 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 shutdown_connections 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 >> 172.21.15.90:0/1931214271 conn(0x7f7c40075d90 msgr2=0x7f7c400fea90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 shutdown_connections 2024-04-06T14:04:27.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.548+0000 7f7c476ee700 1 -- 172.21.15.90:0/1931214271 wait complete. 2024-04-06T14:04:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:27 smithi134 ceph-mon[47549]: pgmap v232: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.0 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-06T14:04:27.947 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:27.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 -- 172.21.15.90:0/3221597277 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 msgr2=0x7f1b6c100120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 --2- 172.21.15.90:0/3221597277 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c100120 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c009990 tx=0x7f1b5c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 -- 172.21.15.90:0/3221597277 shutdown_connections 2024-04-06T14:04:27.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 --2- 172.21.15.90:0/3221597277 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b6c100660 0x7f1b6c10cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 --2- 172.21.15.90:0/3221597277 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b6c105d50 0x7f1b6c106130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 --2- 172.21.15.90:0/3221597277 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c100120 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 -- 172.21.15.90:0/3221597277 >> 172.21.15.90:0/3221597277 conn(0x7f1b6c076c80 msgr2=0x7f1b6c077090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 -- 172.21.15.90:0/3221597277 shutdown_connections 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.955+0000 7f1b735c9700 1 -- 172.21.15.90:0/3221597277 wait complete. 2024-04-06T14:04:27.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 Processor -- start 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 -- start start 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c197160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b6c100660 0x7f1b6c1976a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b6c105d50 0x7f1b6c19ba00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c19c010 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c19c180 con 0x7f1b6c100660 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b735c9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b6c19c340 con 0x7f1b6c105d50 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.956+0000 7f1b71365700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c197160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c197160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45362/0 (socket says 172.21.15.90:45362) 2024-04-06T14:04:27.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 -- 172.21.15.90:0/961654829 learned_addr learned my addr 172.21.15.90:0/961654829 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b70b64700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b6c100660 0x7f1b6c1976a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 -- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b6c105d50 msgr2=0x7f1b6c19ba00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b6c105d50 0x7f1b6c19ba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 -- 172.21.15.90:0/961654829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b6c100660 msgr2=0x7f1b6c1976a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b6c100660 0x7f1b6c1976a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b5c0095f0 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b71365700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c197160 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f1b6800e9a0 tx=0x7f1b6800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b68009790 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b6c19c620 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b6c1a3e80 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1b6800cbd0 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.957+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b68018680 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.958+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f1b50000fc0 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.960+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f1b6800cd40 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.960+0000 7f1b627fc700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1b580a1230 0x7f1b580a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.960+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f1b68014070 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b627fc700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] conn(0x7f1b580aaa10 0x7f1b580ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f1b580ad4e0 con 0x7f1b580aaa10 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1b680cd580 con 0x7f1b6c0ffca0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b70b64700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1b580a1230 0x7f1b580a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b71b66700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] conn(0x7f1b580aaa10 0x7f1b580ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b70b64700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1b580a1230 0x7f1b580a36f0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c006010 tx=0x7f1b5c00b260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.961+0000 7f1b71b66700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] conn(0x7f1b580aaa10 0x7f1b580ace30 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:27.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.962+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== osd.5 v2:172.21.15.134:6808/1329717087 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f1b580ad4e0 con 0x7f1b580aaa10 2024-04-06T14:04:27.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.980+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 --> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f1b50002d40 con 0x7f1b580aaa10 2024-04-06T14:04:27.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b627fc700 1 -- 172.21.15.90:0/961654829 <== osd.5 v2:172.21.15.134:6808/1329717087 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f1b50002d40 con 0x7f1b580aaa10 2024-04-06T14:04:27.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] conn(0x7f1b580aaa10 msgr2=0x7f1b580ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] conn(0x7f1b580aaa10 0x7f1b580ace30 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:27.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1b580a1230 msgr2=0x7f1b580a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f1b580a1230 0x7f1b580a36f0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f1b5c006010 tx=0x7f1b5c00b260 comp rx=0 tx=0).stop 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 msgr2=0x7f1b6c197160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b735c9700 1 --2- 172.21.15.90:0/961654829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b6c0ffca0 0x7f1b6c197160 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f1b6800e9a0 tx=0x7f1b6800ed60 comp rx=0 tx=0).stop 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.981+0000 7f1b71b66700 1 -- 172.21.15.90:0/961654829 reap_dead start 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.982+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 shutdown_connections 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.982+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 >> 172.21.15.90:0/961654829 conn(0x7f1b6c076c80 msgr2=0x7f1b6c0feb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.982+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 shutdown_connections 2024-04-06T14:04:27.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:27.982+0000 7f1b735c9700 1 -- 172.21.15.90:0/961654829 wait complete. 2024-04-06T14:04:28.239 INFO:teuthology.orchestra.run.smithi090.stdout:141733920808 2024-04-06T14:04:28.240 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-06T14:04:28.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:28 smithi090 ceph-mon[43420]: pgmap v233: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-06T14:04:28.726 INFO:teuthology.orchestra.run.smithi090.stdout:163208757284 2024-04-06T14:04:28.726 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-06T14:04:28.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:28 smithi105 ceph-mon[46969]: pgmap v233: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-06T14:04:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:28 smithi134 ceph-mon[47549]: pgmap v233: 97 pgs: 97 active+clean; 635 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.6 KiB/s rd, 1.2 KiB/s wr, 6 op/s 2024-04-06T14:04:28.928 INFO:teuthology.orchestra.run.smithi090.stdout:210453397533 2024-04-06T14:04:28.929 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-06T14:04:29.290 INFO:teuthology.orchestra.run.smithi090.stdout:270582939670 2024-04-06T14:04:29.291 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.9 2024-04-06T14:04:29.380 INFO:teuthology.orchestra.run.smithi090.stdout:193273528351 2024-04-06T14:04:29.380 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-06T14:04:30.071 INFO:teuthology.orchestra.run.smithi090.stdout:55834574898 2024-04-06T14:04:30.071 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-06T14:04:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:30.441 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:30.553 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- 172.21.15.90:0/1619111369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 msgr2=0x7f936c101260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 --2- 172.21.15.90:0/1619111369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c101260 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f935c009990 tx=0x7f935c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- 172.21.15.90:0/1619111369 shutdown_connections 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 --2- 172.21.15.90:0/1619111369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f936c1017a0 0x7f936c10dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 --2- 172.21.15.90:0/1619111369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f936c106e40 0x7f936c107220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 --2- 172.21.15.90:0/1619111369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c101260 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- 172.21.15.90:0/1619111369 >> 172.21.15.90:0/1619111369 conn(0x7f936c077fa0 msgr2=0x7f936c0783b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:30.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- 172.21.15.90:0/1619111369 shutdown_connections 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- 172.21.15.90:0/1619111369 wait complete. 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 Processor -- start 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.678+0000 7f9373d55700 1 -- start start 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c0730b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f936c1017a0 0x7f936c0735f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f936c106e40 0x7f936c073b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f936c1a5590 con 0x7f936c100e00 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f936c1a5700 con 0x7f936c1017a0 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f936c1a58c0 con 0x7f936c106e40 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c0730b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c0730b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45368/0 (socket says 172.21.15.90:45368) 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 -- 172.21.15.90:0/286166728 learned_addr learned my addr 172.21.15.90:0/286166728 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:30.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f93712f0700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f936c1017a0 0x7f936c0735f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f93722f2700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f936c106e40 0x7f936c073b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 -- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f936c106e40 msgr2=0x7f936c073b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f936c106e40 0x7f936c073b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 -- 172.21.15.90:0/286166728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f936c1017a0 msgr2=0x7f936c0735f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f936c1017a0 0x7f936c0735f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f935c0095f0 con 0x7f936c100e00 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9371af1700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c0730b0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f936800b530 tx=0x7f936800b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9368011830 con 0x7f936c100e00 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9368011e50 con 0x7f936c100e00 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f936800f2e0 con 0x7f936c100e00 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f936c1a5ba0 con 0x7f936c100e00 2024-04-06T14:04:30.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.679+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f936c1a6070 con 0x7f936c100e00 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.680+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f936c1024f0 con 0x7f936c100e00 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.681+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9368010380 con 0x7f936c100e00 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f9362ffd700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f93580a1230 0x7f93580a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f93712f0700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f93580a1230 0x7f93580a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f93680cc8f0 con 0x7f936c100e00 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f9362ffd700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] conn(0x7f93580aaa10 0x7f93580ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f93580ad4e0 con 0x7f93580aaa10 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7f93680ccd90 con 0x7f936c100e00 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.682+0000 7f93712f0700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f93580a1230 0x7f93580a36f0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f935c0098e0 tx=0x7f935c005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.683+0000 7f93722f2700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] conn(0x7f93580aaa10 0x7f93580ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:30.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.683+0000 7f93722f2700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] conn(0x7f93580aaa10 0x7f93580ace30 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.11 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:30.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.684+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== osd.11 v2:172.21.15.134:6824/1707163828 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7f93580ad4e0 con 0x7f93580aaa10 2024-04-06T14:04:30.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.699+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 --> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f936c110f80 con 0x7f93580aaa10 2024-04-06T14:04:30.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.700+0000 7f9362ffd700 1 -- 172.21.15.90:0/286166728 <== osd.11 v2:172.21.15.134:6824/1707163828 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f936c110f80 con 0x7f93580aaa10 2024-04-06T14:04:30.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.700+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] conn(0x7f93580aaa10 msgr2=0x7f93580ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] conn(0x7f93580aaa10 0x7f93580ace30 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f93580a1230 msgr2=0x7f93580a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f93580a1230 0x7f93580a36f0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f935c0098e0 tx=0x7f935c005960 comp rx=0 tx=0).stop 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 msgr2=0x7f936c0730b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 --2- 172.21.15.90:0/286166728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f936c100e00 0x7f936c0730b0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f936800b530 tx=0x7f936800b840 comp rx=0 tx=0).stop 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f93722f2700 1 -- 172.21.15.90:0/286166728 reap_dead start 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 shutdown_connections 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 >> 172.21.15.90:0/286166728 conn(0x7f936c077fa0 msgr2=0x7f936c0ffbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 shutdown_connections 2024-04-06T14:04:30.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:30.701+0000 7f9373d55700 1 -- 172.21.15.90:0/286166728 wait complete. 2024-04-06T14:04:30.753 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:31.448 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:31 smithi090 ceph-mon[43420]: pgmap v234: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-04-06T14:04:31.470 INFO:teuthology.orchestra.run.smithi090.stdout:326417514512 2024-04-06T14:04:31.470 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.11 2024-04-06T14:04:31.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:31 smithi105 ceph-mon[46969]: pgmap v234: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-04-06T14:04:31.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:31 smithi134 ceph-mon[47549]: pgmap v234: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 -- 172.21.15.90:0/3178735076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 msgr2=0x7fd69010dcf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 --2- 172.21.15.90:0/3178735076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69010dcf0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fd68800b0a0 tx=0x7fd68800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 -- 172.21.15.90:0/3178735076 shutdown_connections 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 --2- 172.21.15.90:0/3178735076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69010dcf0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 --2- 172.21.15.90:0/3178735076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd6900ff9d0 0x7fd6900ffdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 --2- 172.21.15.90:0/3178735076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd690100380 0x7fd690105ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 -- 172.21.15.90:0/3178735076 >> 172.21.15.90:0/3178735076 conn(0x7fd690074fa0 msgr2=0x7fd6900753b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 -- 172.21.15.90:0/3178735076 shutdown_connections 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.767+0000 7fd695423700 1 -- 172.21.15.90:0/3178735076 wait complete. 2024-04-06T14:04:32.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 Processor -- start 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 -- start start 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd6900ff9d0 0x7fd6901994a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd690100380 0x7fd6901999e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd69019e350 con 0x7fd6901061f0 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd69019e4c0 con 0x7fd6900ff9d0 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd695423700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd69019e680 con 0x7fd690100380 2024-04-06T14:04:32.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.768+0000 7fd68f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stdout:240518168602 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45382/0 (socket says 172.21.15.90:45382) 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 -- 172.21.15.90:0/14640409 learned_addr learned my addr 172.21.15.90:0/14640409 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68effd700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd6900ff9d0 0x7fd6901994a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68e7fc700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd690100380 0x7fd6901999e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 -- 172.21.15.90:0/14640409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd690100380 msgr2=0x7fd6901999e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd690100380 0x7fd6901999e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 -- 172.21.15.90:0/14640409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd6900ff9d0 msgr2=0x7fd6901994a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd6900ff9d0 0x7fd6901994a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 -- 172.21.15.90:0/14640409 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd688009d00 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd68f7fe700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fd688006130 tx=0x7fd688009170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd68800e040 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd69019e900 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd688003c00 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd68801b690 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.769+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6901a61c0 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.770+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd690103030 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.771+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fd68801b7f0 con 0x7fd6901061f0 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.772+0000 7fd677fff700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 0x7fd6780a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:32.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.772+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fd6880cdc20 con 0x7fd6901061f0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.772+0000 7fd68effd700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 0x7fd6780a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.772+0000 7fd68effd700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 0x7fd6780a3670 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fd68000ac50 tx=0x7fd68000a380 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.776+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd688097490 con 0x7fd6901061f0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.976+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 8} v 0) v1 -- 0x7fd690068f00 con 0x7fd6901061f0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.976+0000 7fd677fff700 1 -- 172.21.15.90:0/14640409 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 8}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7fd68809aab0 con 0x7fd6901061f0 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 msgr2=0x7fd6780a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 0x7fd6780a3670 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fd68000ac50 tx=0x7fd68000a380 comp rx=0 tx=0).stop 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 msgr2=0x7fd69019dd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7fd688006130 tx=0x7fd688009170 comp rx=0 tx=0).stop 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 shutdown_connections 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd6780a11b0 0x7fd6780a3670 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd6901061f0 0x7fd69019dd40 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd6900ff9d0 0x7fd6901994a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 --2- 172.21.15.90:0/14640409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd690100380 0x7fd6901999e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:32.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 >> 172.21.15.90:0/14640409 conn(0x7fd690074fa0 msgr2=0x7fd6900fd720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:32.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.978+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 shutdown_connections 2024-04-06T14:04:32.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:31.979+0000 7fd695423700 1 -- 172.21.15.90:0/14640409 wait complete. 2024-04-06T14:04:32.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:32 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/14640409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-06T14:04:32.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:32 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/14640409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-06T14:04:32.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:32 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/14640409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2024-04-06T14:04:32.866 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:33.084 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168600 got 240518168602 for osd.8 2024-04-06T14:04:33.085 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:33.102 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:33.327 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:33 smithi090 ceph-mon[43420]: pgmap v235: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:33.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- 172.21.15.90:0/93781624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069e30 msgr2=0x7f181810dc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 --2- 172.21.15.90:0/93781624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069e30 0x7f181810dc50 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f181000b0d0 tx=0x7f181000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- 172.21.15.90:0/93781624 shutdown_connections 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 --2- 172.21.15.90:0/93781624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069e30 0x7f181810dc50 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 --2- 172.21.15.90:0/93781624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 0x7f18181072f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 --2- 172.21.15.90:0/93781624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069470 0x7f18180698f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- 172.21.15.90:0/93781624 >> 172.21.15.90:0/93781624 conn(0x7f1818075d90 msgr2=0x7f18180761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- 172.21.15.90:0/93781624 shutdown_connections 2024-04-06T14:04:33.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- 172.21.15.90:0/93781624 wait complete. 2024-04-06T14:04:33.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:33 smithi105 ceph-mon[46969]: pgmap v235: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:33.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 Processor -- start 2024-04-06T14:04:33.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.496+0000 7f181fd12700 1 -- start start 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069e30 0x7f18181977f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 0x7f181819bba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181819c1b0 con 0x7f1818069470 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181819c320 con 0x7f1818106f10 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181819c4e0 con 0x7f1818069e30 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34090/0 (socket says 172.21.15.90:34090) 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 -- 172.21.15.90:0/600791763 learned_addr learned my addr 172.21.15.90:0/600791763 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181d2ad700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069e30 0x7f18181977f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181e2af700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 0x7f181819bba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 -- 172.21.15.90:0/600791763 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069e30 msgr2=0x7f18181977f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069e30 0x7f18181977f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 -- 172.21.15.90:0/600791763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 msgr2=0x7f181819bba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 0x7f181819bba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 -- 172.21.15.90:0/600791763 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1810009d00 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181daae700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f181400d950 tx=0x7f181400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f181400f890 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1814010460 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f181819c7c0 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.497+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f181400e610 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.498+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18181a4020 con 0x7f1818069470 2024-04-06T14:04:33.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.498+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18181093a0 con 0x7f1818069470 2024-04-06T14:04:33.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.499+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f18140108d0 con 0x7f1818069470 2024-04-06T14:04:33.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.500+0000 7f180effd700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 0x7f18040a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.500+0000 7f181d2ad700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 0x7f18040a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.500+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f18140cc5e0 con 0x7f1818069470 2024-04-06T14:04:33.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.500+0000 7f181d2ad700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 0x7f18040a3670 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f1818198810 tx=0x7f1808009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:33.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.504+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f1814095dc0 con 0x7f1818069470 2024-04-06T14:04:33.559 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:33.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/2976104237 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac069100 msgr2=0x7fcfac10dc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/2976104237 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac069100 0x7fcfac10dc70 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fcfa400b0a0 tx=0x7fcfa400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/2976104237 shutdown_connections 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/2976104237 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac069100 0x7fcfac10dc70 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/2976104237 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac106e30 0x7fcfac107210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/2976104237 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac068760 0x7fcfac068bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/2976104237 >> 172.21.15.90:0/2976104237 conn(0x7fcfac075d90 msgr2=0x7fcfac0761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/2976104237 shutdown_connections 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.619+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/2976104237 wait complete. 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 Processor -- start 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 -- start start 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac069100 0x7fcfac197820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac106e30 0x7fcfac19bbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfac19c1e0 con 0x7fcfac068760 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfac19c350 con 0x7fcfac069100 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb2e6b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfac19c510 con 0x7fcfac106e30 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb0c07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb0c07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34114/0 (socket says 172.21.15.90:34114) 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb0c07700 1 -- 172.21.15.90:0/705094702 learned_addr learned my addr 172.21.15.90:0/705094702 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfabfff700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac069100 0x7fcfac197820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.620+0000 7fcfb1408700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac106e30 0x7fcfac19bbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 -- 172.21.15.90:0/705094702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac106e30 msgr2=0x7fcfac19bbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac106e30 0x7fcfac19bbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 -- 172.21.15.90:0/705094702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac069100 msgr2=0x7fcfac197820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac069100 0x7fcfac197820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 -- 172.21.15.90:0/705094702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfa4009d00 con 0x7fcfac068760 2024-04-06T14:04:33.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb0c07700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fcfa000d950 tx=0x7fcfa000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:33.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfa000f890 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfa0010460 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcfac19c7f0 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfa000e610 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.621+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcfac1a4050 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.622+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcfac109350 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.623+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fcfa00108d0 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.624+0000 7fcfa9ffb700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 0x7fcf9c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.624+0000 7fcfabfff700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 0x7fcf9c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.624+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fcfa00cc5e0 con 0x7fcfac068760 2024-04-06T14:04:33.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.624+0000 7fcfabfff700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 0x7fcf9c0a3670 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fcfac198840 tx=0x7fcf98009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:33.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.628+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fcfa0095dc0 con 0x7fcfac068760 2024-04-06T14:04:33.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:33 smithi134 ceph-mon[47549]: pgmap v235: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:33.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.716+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f18181985c0 con 0x7f1818069470 2024-04-06T14:04:33.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.716+0000 7f180effd700 1 -- 172.21.15.90:0/600791763 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f18140993e0 con 0x7f1818069470 2024-04-06T14:04:33.717 INFO:teuthology.orchestra.run.smithi090.stdout:77309411376 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 msgr2=0x7f18040a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 0x7f18040a3670 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f1818198810 tx=0x7f1808009260 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 msgr2=0x7f18181972b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f181400d950 tx=0x7f181400dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 shutdown_connections 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f18040a11b0 0x7f18040a3670 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1818106f10 0x7f181819bba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1818069470 0x7f18181972b0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 --2- 172.21.15.90:0/600791763 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1818069e30 0x7f18181977f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 >> 172.21.15.90:0/600791763 conn(0x7f1818075d90 msgr2=0x7f18180fe9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 shutdown_connections 2024-04-06T14:04:33.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.719+0000 7f181fd12700 1 -- 172.21.15.90:0/600791763 wait complete. 2024-04-06T14:04:33.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.841+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 10} v 0) v1 -- 0x7fcfac1024e0 con 0x7fcfac068760 2024-04-06T14:04:33.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.841+0000 7fcfa9ffb700 1 -- 172.21.15.90:0/705094702 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 10}]=0 v0) v1 ==== 75+0+13 (secure 0 0 0) 0x7fcfa00993e0 con 0x7fcfac068760 2024-04-06T14:04:33.842 INFO:teuthology.orchestra.run.smithi090.stdout:292057776147 2024-04-06T14:04:33.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 msgr2=0x7fcf9c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 0x7fcf9c0a3670 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fcfac198840 tx=0x7fcf98009260 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 msgr2=0x7fcfac1972e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fcfa000d950 tx=0x7fcfa000dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 shutdown_connections 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcf9c0a11b0 0x7fcf9c0a3670 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfac106e30 0x7fcfac19bbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfac068760 0x7fcfac1972e0 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 --2- 172.21.15.90:0/705094702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfac069100 0x7fcfac197820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 >> 172.21.15.90:0/705094702 conn(0x7fcfac075d90 msgr2=0x7fcfac0fea10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:33.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 shutdown_connections 2024-04-06T14:04:33.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:33.844+0000 7fcfb2e6b700 1 -- 172.21.15.90:0/705094702 wait complete. 2024-04-06T14:04:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/600791763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-06T14:04:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/705094702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-06T14:04:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/600791763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-06T14:04:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/705094702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-06T14:04:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/600791763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-06T14:04:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/705094702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 -- 172.21.15.90:0/455157273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f19500ff9b0 msgr2=0x7f19500ffd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 --2- 172.21.15.90:0/455157273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f19500ff9b0 0x7f19500ffd90 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f19440099c0 tx=0x7f1944009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 -- 172.21.15.90:0/455157273 shutdown_connections 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 --2- 172.21.15.90:0/455157273 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19501061d0 0x7f195010dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 --2- 172.21.15.90:0/455157273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f19500ff9b0 0x7f19500ffd90 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 --2- 172.21.15.90:0/455157273 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1950100360 0x7f1950105aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 -- 172.21.15.90:0/455157273 >> 172.21.15.90:0/455157273 conn(0x7f1950075070 msgr2=0x7f1950075480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 -- 172.21.15.90:0/455157273 shutdown_connections 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.696+0000 7f1955d19700 1 -- 172.21.15.90:0/455157273 wait complete. 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 Processor -- start 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 -- start start 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19500ff9b0 0x7f19501970a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f19501061d0 0x7f195019b990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f195019bfa0 con 0x7f1950100360 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f195019c110 con 0x7f19501061d0 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f1955d19700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f195019c2d0 con 0x7f19500ff9b0 2024-04-06T14:04:34.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34138/0 (socket says 172.21.15.90:34138) 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194effd700 1 -- 172.21.15.90:0/341390582 learned_addr learned my addr 172.21.15.90:0/341390582 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194ffff700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f19501061d0 0x7f195019b990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194f7fe700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19500ff9b0 0x7f19501970a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194effd700 1 -- 172.21.15.90:0/341390582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19500ff9b0 msgr2=0x7f19501970a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.697+0000 7f194effd700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19500ff9b0 0x7f19501970a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194effd700 1 -- 172.21.15.90:0/341390582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f19501061d0 msgr2=0x7f195019b990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194effd700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f19501061d0 0x7f195019b990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194effd700 1 -- 172.21.15.90:0/341390582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19440095f0 con 0x7f1950100360 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194effd700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f194000c9b0 tx=0x7f194000ccc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1940007b00 con 0x7f1950100360 2024-04-06T14:04:34.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f195019c5b0 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f194000ce50 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1940018660 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.698+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f19501a3e10 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.699+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f19501093b0 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.700+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f194000f450 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.700+0000 7f194cff9700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 0x7f19380a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.700+0000 7f194f7fe700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 0x7f19380a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.700+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f19400cca50 con 0x7f1950100360 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.701+0000 7f194f7fe700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 0x7f19380a3670 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f1944009f50 tx=0x7f1944005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:34.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.704+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f19400962c0 con 0x7f1950100360 2024-04-06T14:04:34.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.914+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) v1 -- 0x7f19501983b0 con 0x7f1950100360 2024-04-06T14:04:34.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.914+0000 7f194cff9700 1 -- 172.21.15.90:0/341390582 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f1940007e60 con 0x7f1950100360 2024-04-06T14:04:34.915 INFO:teuthology.orchestra.run.smithi090.stdout:193273528353 2024-04-06T14:04:34.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.916+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 msgr2=0x7f19380a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.916+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 0x7f19380a3670 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f1944009f50 tx=0x7f1944005fb0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.916+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 msgr2=0x7f19501975e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.916+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f194000c9b0 tx=0x7f194000ccc0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 shutdown_connections 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f19380a11b0 0x7f19380a3670 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f19501061d0 0x7f195019b990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f19500ff9b0 0x7f19501970a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 --2- 172.21.15.90:0/341390582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1950100360 0x7f19501975e0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 >> 172.21.15.90:0/341390582 conn(0x7f1950075070 msgr2=0x7f19500fe800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:34.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 shutdown_connections 2024-04-06T14:04:34.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:34.917+0000 7f1955d19700 1 -- 172.21.15.90:0/341390582 wait complete. 2024-04-06T14:04:35.384 INFO:tasks.cephadm.ceph_manager.ceph:need seq 292057776145 got 292057776147 for osd.10 2024-04-06T14:04:35.385 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:35 smithi105 ceph-mon[46969]: pgmap v236: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/341390582' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-06T14:04:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:35 smithi090 ceph-mon[43420]: pgmap v236: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/341390582' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-06T14:04:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:35 smithi134 ceph-mon[47549]: pgmap v236: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 426 B/s wr, 6 op/s 2024-04-06T14:04:35.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/341390582' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-06T14:04:35.739 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:36.092 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:36.129 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528351 got 193273528353 for osd.6 2024-04-06T14:04:36.129 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 -- 172.21.15.90:0/431768809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58106e30 msgr2=0x7fdc58107210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 --2- 172.21.15.90:0/431768809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58106e30 0x7fdc58107210 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fdc48009990 tx=0x7fdc48009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 -- 172.21.15.90:0/431768809 shutdown_connections 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 --2- 172.21.15.90:0/431768809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdc58069100 0x7fdc5810dc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 --2- 172.21.15.90:0/431768809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58106e30 0x7fdc58107210 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 --2- 172.21.15.90:0/431768809 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58068760 0x7fdc58068bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 -- 172.21.15.90:0/431768809 >> 172.21.15.90:0/431768809 conn(0x7fdc58075d90 msgr2=0x7fdc580761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:36.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 -- 172.21.15.90:0/431768809 shutdown_connections 2024-04-06T14:04:36.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.601+0000 7fdc5e140700 1 -- 172.21.15.90:0/431768809 wait complete. 2024-04-06T14:04:36.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 Processor -- start 2024-04-06T14:04:36.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 -- start start 2024-04-06T14:04:36.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58069100 0x7fdc581002f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdc58106e30 0x7fdc58100830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc581a5550 con 0x7fdc58068760 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc581a56c0 con 0x7fdc58069100 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc5e140700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc581a5880 con 0x7fdc58106e30 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc577fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc577fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34158/0 (socket says 172.21.15.90:34158) 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.602+0000 7fdc577fe700 1 -- 172.21.15.90:0/3804041368 learned_addr learned my addr 172.21.15.90:0/3804041368 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc56ffd700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58069100 0x7fdc581002f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 -- 172.21.15.90:0/3804041368 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdc58106e30 msgr2=0x7fdc58100830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdc58106e30 0x7fdc58100830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 -- 172.21.15.90:0/3804041368 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58069100 msgr2=0x7fdc581002f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58069100 0x7fdc581002f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 -- 172.21.15.90:0/3804041368 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdc480095f0 con 0x7fdc58068760 2024-04-06T14:04:36.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc577fe700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fdc480095c0 tx=0x7fdc4800f740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:36.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc4801d070 con 0x7fdc58068760 2024-04-06T14:04:36.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdc581a5b00 con 0x7fdc58068760 2024-04-06T14:04:36.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdc480054f0 con 0x7fdc58068760 2024-04-06T14:04:36.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdc581a5f70 con 0x7fdc58068760 2024-04-06T14:04:36.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.603+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc48004be0 con 0x7fdc58068760 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.604+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdc58109350 con 0x7fdc58068760 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.605+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fdc4800fc30 con 0x7fdc58068760 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.606+0000 7fdc54ff9700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 0x7fdc400a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.606+0000 7fdc56ffd700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 0x7fdc400a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.606+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fdc480cd690 con 0x7fdc58068760 2024-04-06T14:04:36.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.606+0000 7fdc56ffd700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 0x7fdc400a36f0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c005fd0 tx=0x7fdc4c005f60 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:36.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.610+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fdc48096e00 con 0x7fdc58068760 2024-04-06T14:04:36.639 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 -- 172.21.15.90:0/3353137342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc0ff9b0 msgr2=0x7f3dfc0ffd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 --2- 172.21.15.90:0/3353137342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc0ffd90 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f3dec0099c0 tx=0x7f3dec009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 -- 172.21.15.90:0/3353137342 shutdown_connections 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 --2- 172.21.15.90:0/3353137342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 0x7f3dfc10dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 --2- 172.21.15.90:0/3353137342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc0ffd90 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 --2- 172.21.15.90:0/3353137342 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc100360 0x7f3dfc105aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 -- 172.21.15.90:0/3353137342 >> 172.21.15.90:0/3353137342 conn(0x7f3dfc075070 msgr2=0x7f3dfc075480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 -- 172.21.15.90:0/3353137342 shutdown_connections 2024-04-06T14:04:36.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.680+0000 7f3e017b9700 1 -- 172.21.15.90:0/3353137342 wait complete. 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 Processor -- start 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 -- start start 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc104c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 0x7f3dfc19db70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc19e170 con 0x7f3dfc100360 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc19e2e0 con 0x7f3dfc0ff9b0 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3e017b9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dfc19e4a0 con 0x7f3dfc1061d0 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3dfa7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.681+0000 7f3dfa7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34176/0 (socket says 172.21.15.90:34176) 2024-04-06T14:04:36.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 -- 172.21.15.90:0/713023603 learned_addr learned my addr 172.21.15.90:0/713023603 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfaffd700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc104c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfb7fe700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 0x7f3dfc19db70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 -- 172.21.15.90:0/713023603 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 msgr2=0x7f3dfc19db70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 0x7f3dfc19db70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 -- 172.21.15.90:0/713023603 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc0ff9b0 msgr2=0x7f3dfc104c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc104c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 -- 172.21.15.90:0/713023603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3dec0095f0 con 0x7f3dfc100360 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3dfa7fc700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f3df002f990 tx=0x7f3df002fca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df0004220 con 0x7f3dfc100360 2024-04-06T14:04:36.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3dfc19e780 con 0x7f3dfc100360 2024-04-06T14:04:36.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3df0007640 con 0x7f3dfc100360 2024-04-06T14:04:36.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dfc19ecd0 con 0x7f3dfc100360 2024-04-06T14:04:36.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.682+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3df0005650 con 0x7f3dfc100360 2024-04-06T14:04:36.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.683+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3de80052f0 con 0x7f3dfc100360 2024-04-06T14:04:36.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.685+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f3df0004380 con 0x7f3dfc100360 2024-04-06T14:04:36.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.686+0000 7f3de3fff700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 0x7f3de40a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:36.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.686+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f3df00eeb40 con 0x7f3dfc100360 2024-04-06T14:04:36.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.687+0000 7f3dfaffd700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 0x7f3de40a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:36.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.687+0000 7f3dfaffd700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 0x7f3de40a3770 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f3dec009f50 tx=0x7f3dec005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:36.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.690+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3df00b8220 con 0x7f3dfc100360 2024-04-06T14:04:36.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.835+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) v1 -- 0x7fdc58101290 con 0x7fdc58068760 2024-04-06T14:04:36.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.838+0000 7fdc54ff9700 1 -- 172.21.15.90:0/3804041368 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7fdc4809a420 con 0x7fdc58068760 2024-04-06T14:04:36.839 INFO:teuthology.orchestra.run.smithi090.stdout:210453397534 2024-04-06T14:04:36.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 msgr2=0x7fdc400a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 0x7fdc400a36f0 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c005fd0 tx=0x7fdc4c005f60 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 msgr2=0x7fdc580ffdb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fdc480095c0 tx=0x7fdc4800f740 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 shutdown_connections 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fdc400a1230 0x7fdc400a36f0 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdc58106e30 0x7fdc58100830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdc58068760 0x7fdc580ffdb0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 --2- 172.21.15.90:0/3804041368 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdc58069100 0x7fdc581002f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 >> 172.21.15.90:0/3804041368 conn(0x7fdc58075d90 msgr2=0x7fdc580feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:36.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 shutdown_connections 2024-04-06T14:04:36.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.841+0000 7fdc5e140700 1 -- 172.21.15.90:0/3804041368 wait complete. 2024-04-06T14:04:36.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.911+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 9} v 0) v1 -- 0x7f3de8005870 con 0x7f3dfc100360 2024-04-06T14:04:36.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.912+0000 7f3de3fff700 1 -- 172.21.15.90:0/713023603 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 9}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f3df0038020 con 0x7f3dfc100360 2024-04-06T14:04:36.912 INFO:teuthology.orchestra.run.smithi090.stdout:270582939672 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 msgr2=0x7f3de40a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 0x7f3de40a3770 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f3dec009f50 tx=0x7f3dec005fb0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 msgr2=0x7f3dfc105170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f3df002f990 tx=0x7f3df002fca0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 shutdown_connections 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3de40a12b0 0x7f3de40a3770 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3dfc1061d0 0x7f3dfc19db70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3dfc0ff9b0 0x7f3dfc104c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 --2- 172.21.15.90:0/713023603 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3dfc100360 0x7f3dfc105170 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:36.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 >> 172.21.15.90:0/713023603 conn(0x7f3dfc075070 msgr2=0x7f3dfc10da50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:36.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 shutdown_connections 2024-04-06T14:04:36.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7f3e017b9700 1 -- 172.21.15.90:0/713023603 wait complete. 2024-04-06T14:04:37.167 INFO:teuthology.orchestra.run.smithi090.stdout:111669149741 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 -- 172.21.15.90:0/3722645071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00ff560 msgr2=0x7fcfe00ff940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 --2- 172.21.15.90:0/3722645071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00ff560 0x7fcfe00ff940 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fcfd0009990 tx=0x7fcfd0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 -- 172.21.15.90:0/3722645071 shutdown_connections 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 --2- 172.21.15.90:0/3722645071 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 0x7fcfe010dd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 --2- 172.21.15.90:0/3722645071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00ff560 0x7fcfe00ff940 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 --2- 172.21.15.90:0/3722645071 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00fff10 0x7fcfe0105a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 -- 172.21.15.90:0/3722645071 >> 172.21.15.90:0/3722645071 conn(0x7fcfe0076280 msgr2=0x7fcfe00fd630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 -- 172.21.15.90:0/3722645071 shutdown_connections 2024-04-06T14:04:37.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.913+0000 7fcfe5706700 1 -- 172.21.15.90:0/3722645071 wait complete. 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.914+0000 7fcfe5706700 1 Processor -- start 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.914+0000 7fcfe5706700 1 -- start start 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00ff560 0x7fcfe0197290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 0x7fcfe019bb80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfe019c190 con 0x7fcfe00fff10 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfe019c300 con 0x7fcfe00ff560 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfe5706700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfe019c4c0 con 0x7fcfe0106180 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34188/0 (socket says 172.21.15.90:34188) 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 -- 172.21.15.90:0/2601954751 learned_addr learned my addr 172.21.15.90:0/2601954751 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:37.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfdeffd700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00ff560 0x7fcfe0197290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfdf7fe700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 0x7fcfe019bb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 -- 172.21.15.90:0/2601954751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 msgr2=0x7fcfe019bb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 0x7fcfe019bb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 -- 172.21.15.90:0/2601954751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00ff560 msgr2=0x7fcfe0197290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00ff560 0x7fcfe0197290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 -- 172.21.15.90:0/2601954751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfd00095f0 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.915+0000 7fcfde7fc700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fcfd400d950 tx=0x7fcfd400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfd400f890 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfd4010460 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfd400e610 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcfe019c7a0 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.916+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcfe01a4000 con 0x7fcfe00fff10 2024-04-06T14:04:37.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.917+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcfe01093e0 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.919+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fcfd40105d0 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.920+0000 7fcfc7fff700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 0x7fcfc80a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.920+0000 7fcfdeffd700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 0x7fcfc80a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.921+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fcfd40cc8a0 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.921+0000 7fcfdeffd700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 0x7fcfc80a3670 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fcfd0000c00 tx=0x7fcfd0005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.928+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fcfd4095f80 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.140+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7fcfe0102620 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.140+0000 7fcfc7fff700 1 -- 172.21.15.90:0/2601954751 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7fcfd4017020 con 0x7fcfe00fff10 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.142+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 msgr2=0x7fcfc80a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.142+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 0x7fcfc80a3670 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fcfd0000c00 tx=0x7fcfd0005960 comp rx=0 tx=0).stop 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.142+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 msgr2=0x7fcfe01977d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:37.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.142+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fcfd400d950 tx=0x7fcfd400dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 shutdown_connections 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fcfc80a11b0 0x7fcfc80a3670 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcfe0106180 0x7fcfe019bb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcfe00ff560 0x7fcfe0197290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 --2- 172.21.15.90:0/2601954751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcfe00fff10 0x7fcfe01977d0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 >> 172.21.15.90:0/2601954751 conn(0x7fcfe0076280 msgr2=0x7fcfe00fd630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 shutdown_connections 2024-04-06T14:04:37.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:37.143+0000 7fcfe5706700 1 -- 172.21.15.90:0/2601954751 wait complete. 2024-04-06T14:04:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:37 smithi105 ceph-mon[46969]: pgmap v237: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:37 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3804041368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-06T14:04:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:37 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/713023603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-06T14:04:37.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:37 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2601954751' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-06T14:04:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:37 smithi090 ceph-mon[43420]: pgmap v237: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:37 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3804041368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-06T14:04:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:37 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/713023603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-06T14:04:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:37 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2601954751' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-06T14:04:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:37 smithi134 ceph-mon[47549]: pgmap v237: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:37 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3804041368' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-06T14:04:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:37 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/713023603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2024-04-06T14:04:37.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:37 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2601954751' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-06T14:04:37.940 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411374 got 77309411376 for osd.2 2024-04-06T14:04:37.940 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:37.989 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149738 got 111669149741 for osd.3 2024-04-06T14:04:37.989 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:38.638 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:38 smithi090 ceph-mon[43420]: pgmap v238: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:38.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:38 smithi105 ceph-mon[46969]: pgmap v238: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.945+0000 7fabe3e71700 1 -- 172.21.15.90:0/2345839174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc100360 msgr2=0x7fabdc105aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.945+0000 7fabe3e71700 1 --2- 172.21.15.90:0/2345839174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc100360 0x7fabdc105aa0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fabcc0099c0 tx=0x7fabcc009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 -- 172.21.15.90:0/2345839174 shutdown_connections 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 --2- 172.21.15.90:0/2345839174 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc1061d0 0x7fabdc10dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 --2- 172.21.15.90:0/2345839174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc0ffd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 --2- 172.21.15.90:0/2345839174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc100360 0x7fabdc105aa0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 -- 172.21.15.90:0/2345839174 >> 172.21.15.90:0/2345839174 conn(0x7fabdc075070 msgr2=0x7fabdc075480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 -- 172.21.15.90:0/2345839174 shutdown_connections 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.946+0000 7fabe3e71700 1 -- 172.21.15.90:0/2345839174 wait complete. 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 Processor -- start 2024-04-06T14:04:38.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 -- start start 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc104bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc100360 0x7fabdc105110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc1061d0 0x7fabdc19a9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabdc19af90 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabdc19b100 con 0x7fabdc1061d0 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe3e71700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fabdc19b2c0 con 0x7fabdc100360 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe1c0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc104bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe1c0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc104bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34204/0 (socket says 172.21.15.90:34204) 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.947+0000 7fabe1c0d700 1 -- 172.21.15.90:0/761000954 learned_addr learned my addr 172.21.15.90:0/761000954 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe240e700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc1061d0 0x7fabdc19a9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe140c700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc100360 0x7fabdc105110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:38.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 -- 172.21.15.90:0/761000954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc100360 msgr2=0x7fabdc105110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc100360 0x7fabdc105110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 -- 172.21.15.90:0/761000954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc1061d0 msgr2=0x7fabdc19a9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc1061d0 0x7fabdc19a9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fabcc0095f0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe140c700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fabdc100360 0x7fabdc105110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe240e700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fabdc1061d0 0x7fabdc19a9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe1c0d700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc104bd0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fabd800d950 tx=0x7fabd800dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabd8009b40 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fabdc19b5a0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fabdc197ac0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fabd8009ca0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.948+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fabd800f620 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.950+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fabc0000fc0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.952+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fabd8010460 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.952+0000 7fabd2ffd700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fabc80a1230 0x7fabc80a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.952+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fabd80cc7a0 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.952+0000 7fabe140c700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fabc80a1230 0x7fabc80a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabd2ffd700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] conn(0x7fabc80aaa10 0x7fabc80ace30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fabc80ad4e0 con 0x7fabc80aaa10 2024-04-06T14:04:38.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_get_version_reply(handle=1 version=81) v2 ==== 24+0+0 (secure 0 0 0) 0x7fabd80cee30 con 0x7fabdc0ff9b0 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabe240e700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] conn(0x7fabc80aaa10 0x7fabc80ace30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabe140c700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fabc80a1230 0x7fabc80a36f0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fabcc000c00 tx=0x7fabcc003680 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.953+0000 7fabe240e700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] conn(0x7fabc80aaa10 0x7fabc80ace30 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.954+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== osd.0 v2:172.21.15.90:6802/1162173917 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25254 (crc 0 0 0) 0x7fabc80ad4e0 con 0x7fabc80aaa10 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.970+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 --> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fabc0002d40 con 0x7fabc80aaa10 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.971+0000 7fabd2ffd700 1 -- 172.21.15.90:0/761000954 <== osd.0 v2:172.21.15.90:6802/1162173917 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fabc0002d40 con 0x7fabc80aaa10 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] conn(0x7fabc80aaa10 msgr2=0x7fabc80ace30 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] conn(0x7fabc80aaa10 0x7fabc80ace30 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fabc80a1230 msgr2=0x7fabc80a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fabc80a1230 0x7fabc80a36f0 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7fabcc000c00 tx=0x7fabcc003680 comp rx=0 tx=0).stop 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 msgr2=0x7fabdc104bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 --2- 172.21.15.90:0/761000954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fabdc0ff9b0 0x7fabdc104bd0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fabd800d950 tx=0x7fabd800dd10 comp rx=0 tx=0).stop 2024-04-06T14:04:38.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe240e700 1 -- 172.21.15.90:0/761000954 reap_dead start 2024-04-06T14:04:38.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 shutdown_connections 2024-04-06T14:04:38.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 >> 172.21.15.90:0/761000954 conn(0x7fabdc075070 msgr2=0x7fabdc10d9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:38.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 shutdown_connections 2024-04-06T14:04:38.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:36.972+0000 7fabe3e71700 1 -- 172.21.15.90:0/761000954 wait complete. 2024-04-06T14:04:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:38 smithi134 ceph-mon[47549]: pgmap v238: 97 pgs: 97 active+clean; 639 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.8 KiB/s rd, 426 B/s wr, 5 op/s 2024-04-06T14:04:39.382 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:39.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 -- 172.21.15.90:0/3621584578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f05200ffca0 msgr2=0x7f0520100120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:39.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 --2- 172.21.15.90:0/3621584578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f05200ffca0 0x7f0520100120 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f051c009990 tx=0x7f051c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 -- 172.21.15.90:0/3621584578 shutdown_connections 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 --2- 172.21.15.90:0/3621584578 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0520100660 0x7f052010cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 --2- 172.21.15.90:0/3621584578 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 0x7f0520106130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 --2- 172.21.15.90:0/3621584578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f05200ffca0 0x7f0520100120 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.948+0000 7f05272aa700 1 -- 172.21.15.90:0/3621584578 >> 172.21.15.90:0/3621584578 conn(0x7f0520076c80 msgr2=0x7f0520077090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.949+0000 7f05272aa700 1 -- 172.21.15.90:0/3621584578 shutdown_connections 2024-04-06T14:04:39.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.949+0000 7f05272aa700 1 -- 172.21.15.90:0/3621584578 wait complete. 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.949+0000 7f05272aa700 1 Processor -- start 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.949+0000 7f05272aa700 1 -- start start 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f05200ffca0 0x7f0520196060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 0x7f052019a900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052019af10 con 0x7f0520100660 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052019b080 con 0x7f05200ffca0 2024-04-06T14:04:39.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052019b240 con 0x7f0520105d50 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34210/0 (socket says 172.21.15.90:34210) 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 -- 172.21.15.90:0/1394656365 learned_addr learned my addr 172.21.15.90:0/1394656365 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0525046700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f05200ffca0 0x7f0520196060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0525847700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 0x7f052019a900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 -- 172.21.15.90:0/1394656365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 msgr2=0x7f052019a900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 0x7f052019a900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 -- 172.21.15.90:0/1394656365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f05200ffca0 msgr2=0x7f0520196060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f05200ffca0 0x7f0520196060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 -- 172.21.15.90:0/1394656365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f051c0095f0 con 0x7f0520100660 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f0524845700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f051c005770 tx=0x7f051c0057a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f051c01d070 con 0x7f0520100660 2024-04-06T14:04:39.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f052019b4c0 con 0x7f0520100660 2024-04-06T14:04:39.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f051c021410 con 0x7f0520100660 2024-04-06T14:04:39.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.950+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f051c017650 con 0x7f0520100660 2024-04-06T14:04:39.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.951+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f05201a2d80 con 0x7f0520100660 2024-04-06T14:04:39.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.951+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f05201081e0 con 0x7f0520100660 2024-04-06T14:04:39.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.952+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f051c00fb40 con 0x7f0520100660 2024-04-06T14:04:39.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.953+0000 7f05127fc700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 0x7f050c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:39.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.953+0000 7f0525046700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 0x7f050c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:39.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.953+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f051c0cdcf0 con 0x7f0520100660 2024-04-06T14:04:39.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.953+0000 7f0525046700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 0x7f050c0a3670 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f0514007be0 tx=0x7f05140073d0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:39.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:39.957+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f051c097560 con 0x7f0520100660 2024-04-06T14:04:40.073 INFO:tasks.cephadm.ceph_manager.ceph:need seq 270582939670 got 270582939672 for osd.9 2024-04-06T14:04:40.073 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:40.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.172+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) v1 -- 0x7f0520197320 con 0x7f0520100660 2024-04-06T14:04:40.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.173+0000 7f05127fc700 1 -- 172.21.15.90:0/1394656365 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f051c09ab80 con 0x7f0520100660 2024-04-06T14:04:40.174 INFO:teuthology.orchestra.run.smithi090.stdout:163208757286 2024-04-06T14:04:40.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 msgr2=0x7f050c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 0x7f050c0a3670 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f0514007be0 tx=0x7f05140073d0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 msgr2=0x7f05201965a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f051c005770 tx=0x7f051c0057a0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 shutdown_connections 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f050c0a11b0 0x7f050c0a3670 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0520105d50 0x7f052019a900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f05200ffca0 0x7f0520196060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 --2- 172.21.15.90:0/1394656365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0520100660 0x7f05201965a0 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 >> 172.21.15.90:0/1394656365 conn(0x7f0520076c80 msgr2=0x7f05200fe990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:40.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 shutdown_connections 2024-04-06T14:04:40.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.175+0000 7f05272aa700 1 -- 172.21.15.90:0/1394656365 wait complete. 2024-04-06T14:04:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:40 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1394656365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-06T14:04:40.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:40 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1394656365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-06T14:04:40.548 INFO:tasks.cephadm.ceph_manager.ceph:need seq 210453397533 got 210453397534 for osd.7 2024-04-06T14:04:40.548 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:40.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:40 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1394656365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 -- 172.21.15.90:0/1637881902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4100950 msgr2=0x7ffac41048f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 --2- 172.21.15.90:0/1637881902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4100950 0x7ffac41048f0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7ffab8009990 tx=0x7ffab8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 -- 172.21.15.90:0/1637881902 shutdown_connections 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 --2- 172.21.15.90:0/1637881902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4104fd0 0x7ffac410cba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 --2- 172.21.15.90:0/1637881902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 0x7ffac4100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 --2- 172.21.15.90:0/1637881902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4100950 0x7ffac41048f0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 -- 172.21.15.90:0/1637881902 >> 172.21.15.90:0/1637881902 conn(0x7ffac4075d90 msgr2=0x7ffac40761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 -- 172.21.15.90:0/1637881902 shutdown_connections 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.803+0000 7ffac95fc700 1 -- 172.21.15.90:0/1637881902 wait complete. 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 Processor -- start 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 -- start start 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 0x7ffac4195e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4100950 0x7ffac4196360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffac41a5570 con 0x7ffac4104fd0 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffac41a56e0 con 0x7ffac40fffa0 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac95fc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffac41a58a0 con 0x7ffac4100950 2024-04-06T14:04:40.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.804+0000 7ffac37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34220/0 (socket says 172.21.15.90:34220) 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 -- 172.21.15.90:0/24283481 learned_addr learned my addr 172.21.15.90:0/24283481 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac2ffd700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 0x7ffac4195e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac27fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4100950 0x7ffac4196360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 -- 172.21.15.90:0/24283481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4100950 msgr2=0x7ffac4196360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4100950 0x7ffac4196360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 -- 172.21.15.90:0/24283481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 msgr2=0x7ffac4195e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 0x7ffac4195e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 -- 172.21.15.90:0/24283481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffab80095f0 con 0x7ffac4104fd0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac37fe700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7ffabc007d70 tx=0x7ffabc00a340 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffabc00ac90 con 0x7ffac4104fd0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffabc00adf0 con 0x7ffac4104fd0 2024-04-06T14:04:40.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffac41a5b80 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffabc0115e0 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.805+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffac41a6050 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.806+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffac41081d0 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.807+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7ffabc012420 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.808+0000 7ffaabfff700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 0x7ffaac0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.808+0000 7ffac2ffd700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 0x7ffaac0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.808+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7ffabc0cc8e0 con 0x7ffac4104fd0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.809+0000 7ffac2ffd700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 0x7ffaac0a36f0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7ffab4005950 tx=0x7ffab400b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:40.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:40.813+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ffabc0961d0 con 0x7ffac4104fd0 2024-04-06T14:04:41.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.010+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) v1 -- 0x7ffac4069a10 con 0x7ffac4104fd0 2024-04-06T14:04:41.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.010+0000 7ffaabfff700 1 -- 172.21.15.90:0/24283481 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7ffabc016020 con 0x7ffac4104fd0 2024-04-06T14:04:41.011 INFO:teuthology.orchestra.run.smithi090.stdout:141733920810 2024-04-06T14:04:41.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 msgr2=0x7ffaac0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:41.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 0x7ffaac0a36f0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7ffab4005950 tx=0x7ffab400b410 comp rx=0 tx=0).stop 2024-04-06T14:04:41.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 msgr2=0x7ffac41968a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:41.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7ffabc007d70 tx=0x7ffabc00a340 comp rx=0 tx=0).stop 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 shutdown_connections 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ffaac0a1230 0x7ffaac0a36f0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffac4104fd0 0x7ffac41968a0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffac40fffa0 0x7ffac4195e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 --2- 172.21.15.90:0/24283481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffac4100950 0x7ffac4196360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 >> 172.21.15.90:0/24283481 conn(0x7ffac4075d90 msgr2=0x7ffac40fe180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 shutdown_connections 2024-04-06T14:04:41.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:41.013+0000 7ffac95fc700 1 -- 172.21.15.90:0/24283481 wait complete. 2024-04-06T14:04:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:41 smithi090 ceph-mon[43420]: pgmap v239: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 597 B/s wr, 6 op/s 2024-04-06T14:04:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:41 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/24283481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-06T14:04:41.483 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757284 got 163208757286 for osd.5 2024-04-06T14:04:41.484 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:41.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:41 smithi105 ceph-mon[46969]: pgmap v239: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 597 B/s wr, 6 op/s 2024-04-06T14:04:41.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:41 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/24283481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-06T14:04:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:41 smithi134 ceph-mon[47549]: pgmap v239: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.8 KiB/s rd, 597 B/s wr, 6 op/s 2024-04-06T14:04:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:41 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/24283481' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-06T14:04:41.815 INFO:teuthology.orchestra.run.smithi090.stdout:34359738422 2024-04-06T14:04:41.815 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-06T14:04:42.509 INFO:tasks.cephadm.ceph_manager.ceph:need seq 141733920808 got 141733920810 for osd.4 2024-04-06T14:04:42.510 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:42.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 -- 172.21.15.90:0/3481585402 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 msgr2=0x7fd17c068bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 --2- 172.21.15.90:0/3481585402 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c068bb0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fd170009990 tx=0x7fd170009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 -- 172.21.15.90:0/3481585402 shutdown_connections 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 --2- 172.21.15.90:0/3481585402 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 0x7fd17c10dc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 --2- 172.21.15.90:0/3481585402 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd17c106dc0 0x7fd17c1071a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 --2- 172.21.15.90:0/3481585402 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c068bb0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 -- 172.21.15.90:0/3481585402 >> 172.21.15.90:0/3481585402 conn(0x7fd17c076cf0 msgr2=0x7fd17c077100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 -- 172.21.15.90:0/3481585402 shutdown_connections 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.801+0000 7fd181a38700 1 -- 172.21.15.90:0/3481585402 wait complete. 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 Processor -- start 2024-04-06T14:04:42.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 -- start start 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 0x7fd17c197630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd17c106dc0 0x7fd17c19b9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd17c19bff0 con 0x7fd17c068730 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd17c19c160 con 0x7fd17c0690f0 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd181a38700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd17c19c320 con 0x7fd17c106dc0 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd17affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd17affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34236/0 (socket says 172.21.15.90:34236) 2024-04-06T14:04:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd17affd700 1 -- 172.21.15.90:0/3167240471 learned_addr learned my addr 172.21.15.90:0/3167240471 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.802+0000 7fd17a7fc700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 0x7fd17c197630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 -- 172.21.15.90:0/3167240471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd17c106dc0 msgr2=0x7fd17c19b9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd17c106dc0 0x7fd17c19b9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 -- 172.21.15.90:0/3167240471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 msgr2=0x7fd17c197630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 0x7fd17c197630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 -- 172.21.15.90:0/3167240471 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1700095f0 con 0x7fd17c068730 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd17affd700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd16c00c9b0 tx=0x7fd16c00ccc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd16c007b00 con 0x7fd17c068730 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd16c007c60 con 0x7fd17c068730 2024-04-06T14:04:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd17c19c600 con 0x7fd17c068730 2024-04-06T14:04:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd16c018660 con 0x7fd17c068730 2024-04-06T14:04:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.803+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd17c1a3e60 con 0x7fd17c068730 2024-04-06T14:04:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.804+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd17c1092e0 con 0x7fd17c068730 2024-04-06T14:04:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.805+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fd16c00f450 con 0x7fd17c068730 2024-04-06T14:04:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.806+0000 7fd180a36700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 0x7fd1640a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.806+0000 7fd17a7fc700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 0x7fd1640a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.806+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fd16c0cc910 con 0x7fd17c068730 2024-04-06T14:04:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.806+0000 7fd17a7fc700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 0x7fd1640a3670 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fd17000b5c0 tx=0x7fd170005960 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:42.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.810+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd16c095ff0 con 0x7fd17c068730 2024-04-06T14:04:42.862 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 -- 172.21.15.90:0/1772610762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 msgr2=0x7f8af410dc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1772610762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af410dc70 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f8aec00b0a0 tx=0x7f8aec00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 -- 172.21.15.90:0/1772610762 shutdown_connections 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1772610762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af410dc70 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1772610762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 0x7f8af4107210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1772610762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 0x7f8af4068bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 -- 172.21.15.90:0/1772610762 >> 172.21.15.90:0/1772610762 conn(0x7f8af4075d90 msgr2=0x7f8af40761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 -- 172.21.15.90:0/1772610762 shutdown_connections 2024-04-06T14:04:42.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.904+0000 7f8af9c94700 1 -- 172.21.15.90:0/1772610762 wait complete. 2024-04-06T14:04:42.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 Processor -- start 2024-04-06T14:04:42.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 -- start start 2024-04-06T14:04:42.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 0x7f8af40ffdb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 0x7f8af4100830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af41a5710 con 0x7f8af4069100 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af41a5880 con 0x7f8af4106e30 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af9c94700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8af41a5a40 con 0x7f8af4068760 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34244/0 (socket says 172.21.15.90:34244) 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.905+0000 7f8af2ffd700 1 -- 172.21.15.90:0/1461528836 learned_addr learned my addr 172.21.15.90:0/1461528836 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af3fff700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 0x7f8af4100830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af37fe700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 0x7f8af40ffdb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 -- 172.21.15.90:0/1461528836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 msgr2=0x7f8af40ffdb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 0x7f8af40ffdb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 -- 172.21.15.90:0/1461528836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 msgr2=0x7f8af4100830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 0x7f8af4100830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:42.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 -- 172.21.15.90:0/1461528836 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8aec009d00 con 0x7f8af4069100 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af2ffd700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f8ae800d950 tx=0x7f8ae800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae8009940 con 0x7f8af4069100 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ae8010460 con 0x7f8af4069100 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8af41a5d20 con 0x7f8af4069100 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ae800f620 con 0x7f8af4069100 2024-04-06T14:04:42.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.906+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8af41a61c0 con 0x7f8af4069100 2024-04-06T14:04:42.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.907+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8af4109350 con 0x7f8af4069100 2024-04-06T14:04:42.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.908+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f8ae80108d0 con 0x7f8af4069100 2024-04-06T14:04:42.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.909+0000 7f8af0ff9700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 0x7f8adc0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:42.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.909+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f8ae80cc810 con 0x7f8af4069100 2024-04-06T14:04:42.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.912+0000 7f8af37fe700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 0x7f8adc0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:42.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.912+0000 7f8af37fe700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 0x7f8adc0a3670 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4005950 tx=0x7f8ae40058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:42.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:42.912+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8ae8095ef0 con 0x7f8af4069100 2024-04-06T14:04:43.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.046+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7fd17c102470 con 0x7fd17c068730 2024-04-06T14:04:43.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.047+0000 7fd180a36700 1 -- 172.21.15.90:0/3167240471 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fd17c102470 con 0x7fd17c068730 2024-04-06T14:04:43.048 INFO:teuthology.orchestra.run.smithi090.stdout:55834574901 2024-04-06T14:04:43.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 msgr2=0x7fd1640a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 0x7fd1640a3670 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7fd17000b5c0 tx=0x7fd170005960 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 msgr2=0x7fd17c1970f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fd16c00c9b0 tx=0x7fd16c00ccc0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 shutdown_connections 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd1640a11b0 0x7fd1640a3670 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd17c106dc0 0x7fd17c19b9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd17c068730 0x7fd17c1970f0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 --2- 172.21.15.90:0/3167240471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd17c0690f0 0x7fd17c197630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 >> 172.21.15.90:0/3167240471 conn(0x7fd17c076cf0 msgr2=0x7fd17c0fe8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 shutdown_connections 2024-04-06T14:04:43.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.050+0000 7fd181a38700 1 -- 172.21.15.90:0/3167240471 wait complete. 2024-04-06T14:04:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.126+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 11} v 0) v1 -- 0x7f8af4101290 con 0x7f8af4069100 2024-04-06T14:04:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.126+0000 7f8af0ff9700 1 -- 172.21.15.90:0/1461528836 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 11}]=0 v0) v1 ==== 75+0+13 (secure 0 0 0) 0x7f8ae8020020 con 0x7f8af4069100 2024-04-06T14:04:43.127 INFO:teuthology.orchestra.run.smithi090.stdout:326417514515 2024-04-06T14:04:43.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 msgr2=0x7f8adc0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:43.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 0x7f8adc0a3670 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8ae4005950 tx=0x7f8ae40058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 msgr2=0x7f8af41002f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:43.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f8ae800d950 tx=0x7f8ae800dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 shutdown_connections 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8adc0a11b0 0x7f8adc0a3670 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8af4106e30 0x7f8af4100830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8af4068760 0x7f8af40ffdb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 --2- 172.21.15.90:0/1461528836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8af4069100 0x7f8af41002f0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 >> 172.21.15.90:0/1461528836 conn(0x7f8af4075d90 msgr2=0x7f8af40fea00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 shutdown_connections 2024-04-06T14:04:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:43.129+0000 7f8af9c94700 1 -- 172.21.15.90:0/1461528836 wait complete. 2024-04-06T14:04:43.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:43 smithi090 ceph-mon[43420]: pgmap v240: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:43.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:43 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3167240471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-06T14:04:43.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:43 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1461528836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-06T14:04:43.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:43 smithi105 ceph-mon[46969]: pgmap v240: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:43.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:43 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3167240471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-06T14:04:43.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:43 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1461528836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-06T14:04:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:43 smithi134 ceph-mon[47549]: pgmap v240: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:43 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3167240471' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-06T14:04:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:43 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1461528836' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2024-04-06T14:04:43.840 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574898 got 55834574901 for osd.1 2024-04-06T14:04:43.840 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:44.330 INFO:tasks.cephadm.ceph_manager.ceph:need seq 326417514512 got 326417514515 for osd.11 2024-04-06T14:04:44.331 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:44.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.470+0000 7f90e80c3700 1 -- 172.21.15.90:0/2488998244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e00ffce0 msgr2=0x7f90e0100160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:44.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.470+0000 7f90e80c3700 1 --2- 172.21.15.90:0/2488998244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e00ffce0 0x7f90e0100160 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f90d4009990 tx=0x7f90d4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- 172.21.15.90:0/2488998244 shutdown_connections 2024-04-06T14:04:44.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- 172.21.15.90:0/2488998244 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e01006a0 0x7f90e010cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- 172.21.15.90:0/2488998244 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e01060f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- 172.21.15.90:0/2488998244 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e00ffce0 0x7f90e0100160 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- 172.21.15.90:0/2488998244 >> 172.21.15.90:0/2488998244 conn(0x7f90e0075d90 msgr2=0x7f90e00761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- 172.21.15.90:0/2488998244 shutdown_connections 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- 172.21.15.90:0/2488998244 wait complete. 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 Processor -- start 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- start start 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e00ffce0 0x7f90e0197240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e019bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90e019c140 con 0x7f90e01006a0 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90e019c2b0 con 0x7f90e00ffce0 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.471+0000 7f90e80c3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90e019c470 con 0x7f90e0105d10 2024-04-06T14:04:44.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51884/0 (socket says 172.21.15.90:51884) 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 -- 172.21.15.90:0/3316656880 learned_addr learned my addr 172.21.15.90:0/3316656880 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e6660700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e019bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 -- 172.21.15.90:0/3316656880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 msgr2=0x7f90e019bb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e019bb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e5e5f700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e00ffce0 0x7f90e0197240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 -- 172.21.15.90:0/3316656880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e00ffce0 msgr2=0x7f90e0197240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e00ffce0 0x7f90e0197240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 -- 172.21.15.90:0/3316656880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90d40095f0 con 0x7f90e01006a0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e6660700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e019bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e565e700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f90d4000c00 tx=0x7f90d400f880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90d401d070 con 0x7f90e01006a0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f90d400fde0 con 0x7f90e01006a0 2024-04-06T14:04:44.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90e019c6f0 con 0x7f90e01006a0 2024-04-06T14:04:44.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90d4017770 con 0x7f90e01006a0 2024-04-06T14:04:44.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.472+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90e01018b0 con 0x7f90e01006a0 2024-04-06T14:04:44.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.473+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90e01013c0 con 0x7f90e01006a0 2024-04-06T14:04:44.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.474+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f90d4022aa0 con 0x7f90e01006a0 2024-04-06T14:04:44.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.475+0000 7f90deffd700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 0x7f90c80a7b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:44.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.475+0000 7f90e5e5f700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 0x7f90c80a7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:44.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.475+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f90d40cdfe0 con 0x7f90e01006a0 2024-04-06T14:04:44.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.475+0000 7f90e5e5f700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 0x7f90c80a7b00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f90d0007dc0 tx=0x7f90d00072f0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:44.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.478+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f90d4097770 con 0x7f90e01006a0 2024-04-06T14:04:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.669+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7f90e004ea80 con 0x7f90e01006a0 2024-04-06T14:04:44.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.669+0000 7f90deffd700 1 -- 172.21.15.90:0/3316656880 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f90d4031090 con 0x7f90e01006a0 2024-04-06T14:04:44.670 INFO:teuthology.orchestra.run.smithi090.stdout:34359738424 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 msgr2=0x7f90c80a7b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 0x7f90c80a7b00 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f90d0007dc0 tx=0x7f90d00072f0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 msgr2=0x7f90e0197780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f90d4000c00 tx=0x7f90d400f880 comp rx=0 tx=0).stop 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 shutdown_connections 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c80a5640 0x7f90c80a7b00 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90e0105d10 0x7f90e019bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90e00ffce0 0x7f90e0197240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 --2- 172.21.15.90:0/3316656880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90e01006a0 0x7f90e0197780 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:44.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 >> 172.21.15.90:0/3316656880 conn(0x7f90e0075d90 msgr2=0x7f90e0068960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:44.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.671+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 shutdown_connections 2024-04-06T14:04:44.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:44.672+0000 7f90e80c3700 1 -- 172.21.15.90:0/3316656880 wait complete. 2024-04-06T14:04:45.041 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738422 got 34359738424 for osd.0 2024-04-06T14:04:45.041 DEBUG:teuthology.parallel:result is None 2024-04-06T14:04:45.041 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-06T14:04:45.041 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-06T14:04:45.267 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:45 smithi090 ceph-mon[43420]: pgmap v241: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:45 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3316656880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-06T14:04:45.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:45 smithi105 ceph-mon[46969]: pgmap v241: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:45 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3316656880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-06T14:04:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:45 smithi134 ceph-mon[47549]: pgmap v241: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:45 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3316656880' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 -- 172.21.15.90:0/1809506492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 msgr2=0x7efd1010cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1809506492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd1010cb00 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7efd0800b0a0 tx=0x7efd0800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 -- 172.21.15.90:0/1809506492 shutdown_connections 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1809506492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd1010cb00 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1809506492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd10105d50 0x7efd10106130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1809506492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd100ffca0 0x7efd10100120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 -- 172.21.15.90:0/1809506492 >> 172.21.15.90:0/1809506492 conn(0x7efd10076c80 msgr2=0x7efd10077090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 -- 172.21.15.90:0/1809506492 shutdown_connections 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.576+0000 7efd17f9d700 1 -- 172.21.15.90:0/1809506492 wait complete. 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 Processor -- start 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 -- start start 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd100ffca0 0x7efd10196230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd10105d50 0x7efd1019ab20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efd1019b130 con 0x7efd10100660 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efd1019b2a0 con 0x7efd10105d50 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd17f9d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efd1019b460 con 0x7efd100ffca0 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd15538700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:46.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd15538700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51910/0 (socket says 172.21.15.90:51910) 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd15538700 1 -- 172.21.15.90:0/1315177101 learned_addr learned my addr 172.21.15.90:0/1315177101 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd15d39700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd100ffca0 0x7efd10196230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.577+0000 7efd1653a700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd10105d50 0x7efd1019ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 -- 172.21.15.90:0/1315177101 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd100ffca0 msgr2=0x7efd10196230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd100ffca0 0x7efd10196230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 -- 172.21.15.90:0/1315177101 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd10105d50 msgr2=0x7efd1019ab20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd10105d50 0x7efd1019ab20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 -- 172.21.15.90:0/1315177101 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efd08009d00 con 0x7efd10100660 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd15538700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7efd04009970 tx=0x7efd04009c80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd04012070 con 0x7efd10100660 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efd1019b740 con 0x7efd10100660 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efd0400d260 con 0x7efd10100660 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efd100691c0 con 0x7efd10100660 2024-04-06T14:04:46.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.578+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd04017830 con 0x7efd10100660 2024-04-06T14:04:46.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.579+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efd10101390 con 0x7efd10100660 2024-04-06T14:04:46.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.580+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7efd04007af0 con 0x7efd10100660 2024-04-06T14:04:46.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.580+0000 7efd02ffd700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 0x7efcfc0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:46.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.581+0000 7efd15d39700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 0x7efcfc0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:46.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.581+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7efd040cdbd0 con 0x7efd10100660 2024-04-06T14:04:46.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.581+0000 7efd15d39700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 0x7efcfc0a36f0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7efd0c007b60 tx=0x7efd0c0073d0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:46.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.585+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efd040972b0 con 0x7efd10100660 2024-04-06T14:04:46.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.772+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7efd1010fc00 con 0x7efcfc0a1230 2024-04-06T14:04:46.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.782+0000 7efd02ffd700 1 -- 172.21.15.90:0/1315177101 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+341878 (secure 0 0 0) 0x7efd1010fc00 con 0x7efcfc0a1230 2024-04-06T14:04:46.788 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:46.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.785+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 msgr2=0x7efcfc0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.785+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 0x7efcfc0a36f0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7efd0c007b60 tx=0x7efd0c0073d0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.785+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 msgr2=0x7efd10196770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.785+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7efd04009970 tx=0x7efd04009c80 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.785+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 shutdown_connections 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efcfc0a1230 0x7efcfc0a36f0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efd10105d50 0x7efd1019ab20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efd100ffca0 0x7efd10196230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 --2- 172.21.15.90:0/1315177101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efd10100660 0x7efd10196770 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 >> 172.21.15.90:0/1315177101 conn(0x7efd10076c80 msgr2=0x7efd100fe960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 shutdown_connections 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:46.786+0000 7efd17f9d700 1 -- 172.21.15.90:0/1315177101 wait complete. 2024-04-06T14:04:46.791 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:04:47.232 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":242,"stamp":"2024-04-06T14:04:46.169420+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655509,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":385,"num_read_kb":314,"num_write":333,"num_write_kb":3046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":271,"ondisk_log_size":271,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":359448,"kb_used_data":10680,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124582376,"statfs":{"total":1151940427776,"available":1151572353024,"internally_reserved":0,"allocated":10936320,"data_stored":6973602,"data_compressed":112659,"data_compressed_allocated":147456,"data_compressed_original":1794048,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1652,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":66,"num_read_kb":37,"num_write":1,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002743"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489854+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:04:01.489854+0000","last_peered":"2024-04-06T14:04:01.489854+0000","last_clean":"2024-04-06T14:04:01.489854+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:04:01.489854+0000","last_undegraded":"2024-04-06T14:04:01.489854+0000","last_fullsized":"2024-04-06T14:04:01.489854+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574988+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:04:02.574988+0000","last_peered":"2024-04-06T14:04:02.574988+0000","last_clean":"2024-04-06T14:04:02.574988+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:04:02.574988+0000","last_undegraded":"2024-04-06T14:04:02.574988+0000","last_fullsized":"2024-04-06T14:04:02.574988+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574769+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:04:02.574769+0000","last_peered":"2024-04-06T14:04:02.574769+0000","last_clean":"2024-04-06T14:04:02.574769+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:04:02.574769+0000","last_undegraded":"2024-04-06T14:04:02.574769+0000","last_fullsized":"2024-04-06T14:04:02.574769+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470214+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:04:01.470214+0000","last_peered":"2024-04-06T14:04:01.470214+0000","last_clean":"2024-04-06T14:04:01.470214+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:04:01.470214+0000","last_undegraded":"2024-04-06T14:04:01.470214+0000","last_fullsized":"2024-04-06T14:04:01.470214+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575085+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:04:02.575085+0000","last_peered":"2024-04-06T14:04:02.575085+0000","last_clean":"2024-04-06T14:04:02.575085+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:04:02.575085+0000","last_undegraded":"2024-04-06T14:04:02.575085+0000","last_fullsized":"2024-04-06T14:04:02.575085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436129+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:04:02.436129+0000","last_peered":"2024-04-06T14:04:02.436129+0000","last_clean":"2024-04-06T14:04:02.436129+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:04:02.436129+0000","last_undegraded":"2024-04-06T14:04:02.436129+0000","last_fullsized":"2024-04-06T14:04:02.436129+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225702+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:04:02.225702+0000","last_peered":"2024-04-06T14:04:02.225702+0000","last_clean":"2024-04-06T14:04:02.225702+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:04:02.225702+0000","last_undegraded":"2024-04-06T14:04:02.225702+0000","last_fullsized":"2024-04-06T14:04:02.225702+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575535+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:04:02.575535+0000","last_peered":"2024-04-06T14:04:02.575535+0000","last_clean":"2024-04-06T14:04:02.575535+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:04:02.575535+0000","last_undegraded":"2024-04-06T14:04:02.575535+0000","last_fullsized":"2024-04-06T14:04:02.575535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575974+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:04:02.575974+0000","last_peered":"2024-04-06T14:04:02.575974+0000","last_clean":"2024-04-06T14:04:02.575974+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:04:02.575974+0000","last_undegraded":"2024-04-06T14:04:02.575974+0000","last_fullsized":"2024-04-06T14:04:02.575974+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436412+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:04:02.436412+0000","last_peered":"2024-04-06T14:04:02.436412+0000","last_clean":"2024-04-06T14:04:02.436412+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:04:02.436412+0000","last_undegraded":"2024-04-06T14:04:02.436412+0000","last_fullsized":"2024-04-06T14:04:02.436412+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225803+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:04:02.225803+0000","last_peered":"2024-04-06T14:04:02.225803+0000","last_clean":"2024-04-06T14:04:02.225803+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:04:02.225803+0000","last_undegraded":"2024-04-06T14:04:02.225803+0000","last_fullsized":"2024-04-06T14:04:02.225803+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470202+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:04:01.470202+0000","last_peered":"2024-04-06T14:04:01.470202+0000","last_clean":"2024-04-06T14:04:01.470202+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:04:01.470202+0000","last_undegraded":"2024-04-06T14:04:01.470202+0000","last_fullsized":"2024-04-06T14:04:01.470202+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575159+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:04:02.575159+0000","last_peered":"2024-04-06T14:04:02.575159+0000","last_clean":"2024-04-06T14:04:02.575159+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:04:02.575159+0000","last_undegraded":"2024-04-06T14:04:02.575159+0000","last_fullsized":"2024-04-06T14:04:02.575159+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574684+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:04:02.574684+0000","last_peered":"2024-04-06T14:04:02.574684+0000","last_clean":"2024-04-06T14:04:02.574684+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:04:02.574684+0000","last_undegraded":"2024-04-06T14:04:02.574684+0000","last_fullsized":"2024-04-06T14:04:02.574684+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574459+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:04:02.574459+0000","last_peered":"2024-04-06T14:04:02.574459+0000","last_clean":"2024-04-06T14:04:02.574459+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:04:02.574459+0000","last_undegraded":"2024-04-06T14:04:02.574459+0000","last_fullsized":"2024-04-06T14:04:02.574459+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575094+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:04:02.575094+0000","last_peered":"2024-04-06T14:04:02.575094+0000","last_clean":"2024-04-06T14:04:02.575094+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:04:02.575094+0000","last_undegraded":"2024-04-06T14:04:02.575094+0000","last_fullsized":"2024-04-06T14:04:02.575094+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489865+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:04:01.489865+0000","last_peered":"2024-04-06T14:04:01.489865+0000","last_clean":"2024-04-06T14:04:01.489865+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:04:01.489865+0000","last_undegraded":"2024-04-06T14:04:01.489865+0000","last_fullsized":"2024-04-06T14:04:01.489865+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437817+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:04:02.437817+0000","last_peered":"2024-04-06T14:04:02.437817+0000","last_clean":"2024-04-06T14:04:02.437817+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:04:02.437817+0000","last_undegraded":"2024-04-06T14:04:02.437817+0000","last_fullsized":"2024-04-06T14:04:02.437817+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672247+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:04:01.672247+0000","last_peered":"2024-04-06T14:04:01.672247+0000","last_clean":"2024-04-06T14:04:01.672247+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:04:01.672247+0000","last_undegraded":"2024-04-06T14:04:01.672247+0000","last_fullsized":"2024-04-06T14:04:01.672247+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":22,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513730+0000","last_change":"2024-04-06T14:04:01.028407+0000","last_active":"2024-04-06T14:04:05.513730+0000","last_peered":"2024-04-06T14:04:05.513730+0000","last_clean":"2024-04-06T14:04:05.513730+0000","last_became_active":"2024-04-06T14:04:01.028036+0000","last_became_peered":"2024-04-06T14:04:01.028036+0000","last_unstale":"2024-04-06T14:04:05.513730+0000","last_undegraded":"2024-04-06T14:04:05.513730+0000","last_fullsized":"2024-04-06T14:04:05.513730+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:08:53.486282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575627+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:04:02.575627+0000","last_peered":"2024-04-06T14:04:02.575627+0000","last_clean":"2024-04-06T14:04:02.575627+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:04:02.575627+0000","last_undegraded":"2024-04-06T14:04:02.575627+0000","last_fullsized":"2024-04-06T14:04:02.575627+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.115761+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:04:04.115761+0000","last_peered":"2024-04-06T14:04:04.115761+0000","last_clean":"2024-04-06T14:04:04.115761+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:04:04.115761+0000","last_undegraded":"2024-04-06T14:04:04.115761+0000","last_fullsized":"2024-04-06T14:04:04.115761+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575409+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:04:02.575409+0000","last_peered":"2024-04-06T14:04:02.575409+0000","last_clean":"2024-04-06T14:04:02.575409+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:04:02.575409+0000","last_undegraded":"2024-04-06T14:04:02.575409+0000","last_fullsized":"2024-04-06T14:04:02.575409+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512073+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:04:05.512073+0000","last_peered":"2024-04-06T14:04:05.512073+0000","last_clean":"2024-04-06T14:04:05.512073+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:04:05.512073+0000","last_undegraded":"2024-04-06T14:04:05.512073+0000","last_fullsized":"2024-04-06T14:04:05.512073+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575221+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:04:02.575221+0000","last_peered":"2024-04-06T14:04:02.575221+0000","last_clean":"2024-04-06T14:04:02.575221+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:04:02.575221+0000","last_undegraded":"2024-04-06T14:04:02.575221+0000","last_fullsized":"2024-04-06T14:04:02.575221+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575477+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:04:02.575477+0000","last_peered":"2024-04-06T14:04:02.575477+0000","last_clean":"2024-04-06T14:04:02.575477+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:04:02.575477+0000","last_undegraded":"2024-04-06T14:04:02.575477+0000","last_fullsized":"2024-04-06T14:04:02.575477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574918+0000","last_change":"2024-04-06T14:04:01.457311+0000","last_active":"2024-04-06T14:04:02.574918+0000","last_peered":"2024-04-06T14:04:02.574918+0000","last_clean":"2024-04-06T14:04:02.574918+0000","last_became_active":"2024-04-06T14:04:01.456954+0000","last_became_peered":"2024-04-06T14:04:01.456954+0000","last_unstale":"2024-04-06T14:04:02.574918+0000","last_undegraded":"2024-04-06T14:04:02.574918+0000","last_fullsized":"2024-04-06T14:04:02.574918+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:22:59.524995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"81'5","reported_seq":56,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:41.568815+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:04:41.568815+0000","last_peered":"2024-04-06T14:04:41.568815+0000","last_clean":"2024-04-06T14:04:41.568815+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:04:41.568815+0000","last_undegraded":"2024-04-06T14:04:41.568815+0000","last_fullsized":"2024-04-06T14:04:41.568815+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575517+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:04:02.575517+0000","last_peered":"2024-04-06T14:04:02.575517+0000","last_clean":"2024-04-06T14:04:02.575517+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:04:02.575517+0000","last_undegraded":"2024-04-06T14:04:02.575517+0000","last_fullsized":"2024-04-06T14:04:02.575517+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512715+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:04:05.512715+0000","last_peered":"2024-04-06T14:04:05.512715+0000","last_clean":"2024-04-06T14:04:05.512715+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:04:05.512715+0000","last_undegraded":"2024-04-06T14:04:05.512715+0000","last_fullsized":"2024-04-06T14:04:05.512715+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574780+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:04:02.574780+0000","last_peered":"2024-04-06T14:04:02.574780+0000","last_clean":"2024-04-06T14:04:02.574780+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:04:02.574780+0000","last_undegraded":"2024-04-06T14:04:02.574780+0000","last_fullsized":"2024-04-06T14:04:02.574780+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.514049+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:04:05.514049+0000","last_peered":"2024-04-06T14:04:05.514049+0000","last_clean":"2024-04-06T14:04:05.514049+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:04:05.514049+0000","last_undegraded":"2024-04-06T14:04:05.514049+0000","last_fullsized":"2024-04-06T14:04:05.514049+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575621+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:04:02.575621+0000","last_peered":"2024-04-06T14:04:02.575621+0000","last_clean":"2024-04-06T14:04:02.575621+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:04:02.575621+0000","last_undegraded":"2024-04-06T14:04:02.575621+0000","last_fullsized":"2024-04-06T14:04:02.575621+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.309221+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:04:03.309221+0000","last_peered":"2024-04-06T14:04:03.309221+0000","last_clean":"2024-04-06T14:04:03.309221+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:04:03.309221+0000","last_undegraded":"2024-04-06T14:04:03.309221+0000","last_fullsized":"2024-04-06T14:04:03.309221+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437700+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:04:02.437700+0000","last_peered":"2024-04-06T14:04:02.437700+0000","last_clean":"2024-04-06T14:04:02.437700+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:04:02.437700+0000","last_undegraded":"2024-04-06T14:04:02.437700+0000","last_fullsized":"2024-04-06T14:04:02.437700+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'3","reported_seq":52,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:41.568153+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:04:41.568153+0000","last_peered":"2024-04-06T14:04:41.568153+0000","last_clean":"2024-04-06T14:04:41.568153+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:04:41.568153+0000","last_undegraded":"2024-04-06T14:04:41.568153+0000","last_fullsized":"2024-04-06T14:04:41.568153+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":37,"num_read_kb":37,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575670+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:04:02.575670+0000","last_peered":"2024-04-06T14:04:02.575670+0000","last_clean":"2024-04-06T14:04:02.575670+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:04:02.575670+0000","last_undegraded":"2024-04-06T14:04:02.575670+0000","last_fullsized":"2024-04-06T14:04:02.575670+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507363+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:04:04.507363+0000","last_peered":"2024-04-06T14:04:04.507363+0000","last_clean":"2024-04-06T14:04:04.507363+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:04:04.507363+0000","last_undegraded":"2024-04-06T14:04:04.507363+0000","last_fullsized":"2024-04-06T14:04:04.507363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575323+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:04:02.575323+0000","last_peered":"2024-04-06T14:04:02.575323+0000","last_clean":"2024-04-06T14:04:02.575323+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:04:02.575323+0000","last_undegraded":"2024-04-06T14:04:02.575323+0000","last_fullsized":"2024-04-06T14:04:02.575323+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"81'4","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.504954+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:04:04.504954+0000","last_peered":"2024-04-06T14:04:04.504954+0000","last_clean":"2024-04-06T14:04:04.504954+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:04:04.504954+0000","last_undegraded":"2024-04-06T14:04:04.504954+0000","last_fullsized":"2024-04-06T14:04:04.504954+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575352+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:04:02.575352+0000","last_peered":"2024-04-06T14:04:02.575352+0000","last_clean":"2024-04-06T14:04:02.575352+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:04:02.575352+0000","last_undegraded":"2024-04-06T14:04:02.575352+0000","last_fullsized":"2024-04-06T14:04:02.575352+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'28","reported_seq":122,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:42.580477+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:04:42.580477+0000","last_peered":"2024-04-06T14:04:42.580477+0000","last_clean":"2024-04-06T14:04:42.580477+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:04:42.580477+0000","last_undegraded":"2024-04-06T14:04:42.580477+0000","last_fullsized":"2024-04-06T14:04:42.580477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61000,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":82,"num_read_kb":100,"num_write":29,"num_write_kb":69,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575168+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:04:02.575168+0000","last_peered":"2024-04-06T14:04:02.575168+0000","last_clean":"2024-04-06T14:04:02.575168+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:04:02.575168+0000","last_undegraded":"2024-04-06T14:04:02.575168+0000","last_fullsized":"2024-04-06T14:04:02.575168+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":19,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513503+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:04:05.513503+0000","last_peered":"2024-04-06T14:04:05.513503+0000","last_clean":"2024-04-06T14:04:05.513503+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:04:05.513503+0000","last_undegraded":"2024-04-06T14:04:05.513503+0000","last_fullsized":"2024-04-06T14:04:05.513503+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574857+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:04:02.574857+0000","last_peered":"2024-04-06T14:04:02.574857+0000","last_clean":"2024-04-06T14:04:02.574857+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:04:02.574857+0000","last_undegraded":"2024-04-06T14:04:02.574857+0000","last_fullsized":"2024-04-06T14:04:02.574857+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.506929+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:04:04.506929+0000","last_peered":"2024-04-06T14:04:04.506929+0000","last_clean":"2024-04-06T14:04:04.506929+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:04:04.506929+0000","last_undegraded":"2024-04-06T14:04:04.506929+0000","last_fullsized":"2024-04-06T14:04:04.506929+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225762+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:04:02.225762+0000","last_peered":"2024-04-06T14:04:02.225762+0000","last_clean":"2024-04-06T14:04:02.225762+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:04:02.225762+0000","last_undegraded":"2024-04-06T14:04:02.225762+0000","last_fullsized":"2024-04-06T14:04:02.225762+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'4","reported_seq":54,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:39.566905+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:04:39.566905+0000","last_peered":"2024-04-06T14:04:39.566905+0000","last_clean":"2024-04-06T14:04:39.566905+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:04:39.566905+0000","last_undegraded":"2024-04-06T14:04:39.566905+0000","last_fullsized":"2024-04-06T14:04:39.566905+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":933,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225612+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:04:02.225612+0000","last_peered":"2024-04-06T14:04:02.225612+0000","last_clean":"2024-04-06T14:04:02.225612+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:04:02.225612+0000","last_undegraded":"2024-04-06T14:04:02.225612+0000","last_fullsized":"2024-04-06T14:04:02.225612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"81'15","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513774+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:04:05.513774+0000","last_peered":"2024-04-06T14:04:05.513774+0000","last_clean":"2024-04-06T14:04:05.513774+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:04:05.513774+0000","last_undegraded":"2024-04-06T14:04:05.513774+0000","last_fullsized":"2024-04-06T14:04:05.513774+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574613+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:04:02.574613+0000","last_peered":"2024-04-06T14:04:02.574613+0000","last_clean":"2024-04-06T14:04:02.574613+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:04:02.574613+0000","last_undegraded":"2024-04-06T14:04:02.574613+0000","last_fullsized":"2024-04-06T14:04:02.574613+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'2","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:06.516280+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:04:06.516280+0000","last_peered":"2024-04-06T14:04:06.516280+0000","last_clean":"2024-04-06T14:04:06.516280+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:04:06.516280+0000","last_undegraded":"2024-04-06T14:04:06.516280+0000","last_fullsized":"2024-04-06T14:04:06.516280+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489844+0000","last_change":"2024-04-06T14:04:01.455152+0000","last_active":"2024-04-06T14:04:01.489844+0000","last_peered":"2024-04-06T14:04:01.489844+0000","last_clean":"2024-04-06T14:04:01.489844+0000","last_became_active":"2024-04-06T14:04:01.454827+0000","last_became_peered":"2024-04-06T14:04:01.454827+0000","last_unstale":"2024-04-06T14:04:01.489844+0000","last_undegraded":"2024-04-06T14:04:01.489844+0000","last_fullsized":"2024-04-06T14:04:01.489844+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:55:26.359159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.438338+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:04:01.438338+0000","last_peered":"2024-04-06T14:04:01.438338+0000","last_clean":"2024-04-06T14:04:01.438338+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:04:01.438338+0000","last_undegraded":"2024-04-06T14:04:01.438338+0000","last_fullsized":"2024-04-06T14:04:01.438338+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"81'1","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.200634+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:04:03.200634+0000","last_peered":"2024-04-06T14:04:03.200634+0000","last_clean":"2024-04-06T14:04:03.200634+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:04:03.200634+0000","last_undegraded":"2024-04-06T14:04:03.200634+0000","last_fullsized":"2024-04-06T14:04:03.200634+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576581+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:04:02.576581+0000","last_peered":"2024-04-06T14:04:02.576581+0000","last_clean":"2024-04-06T14:04:02.576581+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:04:02.576581+0000","last_undegraded":"2024-04-06T14:04:02.576581+0000","last_fullsized":"2024-04-06T14:04:02.576581+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436536+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:04:02.436536+0000","last_peered":"2024-04-06T14:04:02.436536+0000","last_clean":"2024-04-06T14:04:02.436536+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:04:02.436536+0000","last_undegraded":"2024-04-06T14:04:02.436536+0000","last_fullsized":"2024-04-06T14:04:02.436536+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575047+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:04:02.575047+0000","last_peered":"2024-04-06T14:04:02.575047+0000","last_clean":"2024-04-06T14:04:02.575047+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:04:02.575047+0000","last_undegraded":"2024-04-06T14:04:02.575047+0000","last_fullsized":"2024-04-06T14:04:02.575047+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'4","reported_seq":55,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:43.572538+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:04:43.572538+0000","last_peered":"2024-04-06T14:04:43.572538+0000","last_clean":"2024-04-06T14:04:43.572538+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:04:43.572538+0000","last_undegraded":"2024-04-06T14:04:43.572538+0000","last_fullsized":"2024-04-06T14:04:43.572538+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":39,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575189+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:04:02.575189+0000","last_peered":"2024-04-06T14:04:02.575189+0000","last_clean":"2024-04-06T14:04:02.575189+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:04:02.575189+0000","last_undegraded":"2024-04-06T14:04:02.575189+0000","last_fullsized":"2024-04-06T14:04:02.575189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:06.515167+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:04:06.515167+0000","last_peered":"2024-04-06T14:04:06.515167+0000","last_clean":"2024-04-06T14:04:06.515167+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:04:06.515167+0000","last_undegraded":"2024-04-06T14:04:06.515167+0000","last_fullsized":"2024-04-06T14:04:06.515167+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":334,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575511+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:04:02.575511+0000","last_peered":"2024-04-06T14:04:02.575511+0000","last_clean":"2024-04-06T14:04:02.575511+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:04:02.575511+0000","last_undegraded":"2024-04-06T14:04:02.575511+0000","last_fullsized":"2024-04-06T14:04:02.575511+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574458+0000","last_change":"2024-04-06T14:04:01.457132+0000","last_active":"2024-04-06T14:04:02.574458+0000","last_peered":"2024-04-06T14:04:02.574458+0000","last_clean":"2024-04-06T14:04:02.574458+0000","last_became_active":"2024-04-06T14:04:01.456782+0000","last_became_peered":"2024-04-06T14:04:01.456782+0000","last_unstale":"2024-04-06T14:04:02.574458+0000","last_undegraded":"2024-04-06T14:04:02.574458+0000","last_fullsized":"2024-04-06T14:04:02.574458+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:34:37.759113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512264+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:04:05.512264+0000","last_peered":"2024-04-06T14:04:05.512264+0000","last_clean":"2024-04-06T14:04:05.512264+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:04:05.512264+0000","last_undegraded":"2024-04-06T14:04:05.512264+0000","last_fullsized":"2024-04-06T14:04:05.512264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"81'1","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.505412+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:04:04.505412+0000","last_peered":"2024-04-06T14:04:04.505412+0000","last_clean":"2024-04-06T14:04:04.505412+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:04:04.505412+0000","last_undegraded":"2024-04-06T14:04:04.505412+0000","last_fullsized":"2024-04-06T14:04:04.505412+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574508+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:04:02.574508+0000","last_peered":"2024-04-06T14:04:02.574508+0000","last_clean":"2024-04-06T14:04:02.574508+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:04:02.574508+0000","last_undegraded":"2024-04-06T14:04:02.574508+0000","last_fullsized":"2024-04-06T14:04:02.574508+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'5","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:10.432735+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:04:10.432735+0000","last_peered":"2024-04-06T14:04:10.432735+0000","last_clean":"2024-04-06T14:04:10.432735+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:04:10.432735+0000","last_undegraded":"2024-04-06T14:04:10.432735+0000","last_fullsized":"2024-04-06T14:04:10.432735+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575544+0000","last_change":"2024-04-06T14:04:01.461874+0000","last_active":"2024-04-06T14:04:02.575544+0000","last_peered":"2024-04-06T14:04:02.575544+0000","last_clean":"2024-04-06T14:04:02.575544+0000","last_became_active":"2024-04-06T14:04:01.461652+0000","last_became_peered":"2024-04-06T14:04:01.461652+0000","last_unstale":"2024-04-06T14:04:02.575544+0000","last_undegraded":"2024-04-06T14:04:02.575544+0000","last_fullsized":"2024-04-06T14:04:02.575544+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:01:58.862215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507264+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:04:04.507264+0000","last_peered":"2024-04-06T14:04:04.507264+0000","last_clean":"2024-04-06T14:04:04.507264+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:04:04.507264+0000","last_undegraded":"2024-04-06T14:04:04.507264+0000","last_fullsized":"2024-04-06T14:04:04.507264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575075+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:04:02.575075+0000","last_peered":"2024-04-06T14:04:02.575075+0000","last_clean":"2024-04-06T14:04:02.575075+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:04:02.575075+0000","last_undegraded":"2024-04-06T14:04:02.575075+0000","last_fullsized":"2024-04-06T14:04:02.575075+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507554+0000","last_change":"2024-04-06T14:04:01.035529+0000","last_active":"2024-04-06T14:04:04.507554+0000","last_peered":"2024-04-06T14:04:04.507554+0000","last_clean":"2024-04-06T14:04:04.507554+0000","last_became_active":"2024-04-06T14:04:01.035205+0000","last_became_peered":"2024-04-06T14:04:01.035205+0000","last_unstale":"2024-04-06T14:04:04.507554+0000","last_undegraded":"2024-04-06T14:04:04.507554+0000","last_fullsized":"2024-04-06T14:04:04.507554+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:59:17.605780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574775+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:04:02.574775+0000","last_peered":"2024-04-06T14:04:02.574775+0000","last_clean":"2024-04-06T14:04:02.574775+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:04:02.574775+0000","last_undegraded":"2024-04-06T14:04:02.574775+0000","last_fullsized":"2024-04-06T14:04:02.574775+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.506884+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:04:04.506884+0000","last_peered":"2024-04-06T14:04:04.506884+0000","last_clean":"2024-04-06T14:04:04.506884+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:04:04.506884+0000","last_undegraded":"2024-04-06T14:04:04.506884+0000","last_fullsized":"2024-04-06T14:04:04.506884+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489773+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:04:01.489773+0000","last_peered":"2024-04-06T14:04:01.489773+0000","last_clean":"2024-04-06T14:04:01.489773+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:04:01.489773+0000","last_undegraded":"2024-04-06T14:04:01.489773+0000","last_fullsized":"2024-04-06T14:04:01.489773+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.560630+0000","last_change":"2024-04-06T14:04:01.032612+0000","last_active":"2024-04-06T14:04:02.560630+0000","last_peered":"2024-04-06T14:04:02.560630+0000","last_clean":"2024-04-06T14:04:02.560630+0000","last_became_active":"2024-04-06T14:04:01.032115+0000","last_became_peered":"2024-04-06T14:04:01.032115+0000","last_unstale":"2024-04-06T14:04:02.560630+0000","last_undegraded":"2024-04-06T14:04:02.560630+0000","last_fullsized":"2024-04-06T14:04:02.560630+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:40.607887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575432+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:04:02.575432+0000","last_peered":"2024-04-06T14:04:02.575432+0000","last_clean":"2024-04-06T14:04:02.575432+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:04:02.575432+0000","last_undegraded":"2024-04-06T14:04:02.575432+0000","last_fullsized":"2024-04-06T14:04:02.575432+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"81'4","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.508618+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:04:04.508618+0000","last_peered":"2024-04-06T14:04:04.508618+0000","last_clean":"2024-04-06T14:04:04.508618+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:04:04.508618+0000","last_undegraded":"2024-04-06T14:04:04.508618+0000","last_fullsized":"2024-04-06T14:04:04.508618+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225742+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:04:02.225742+0000","last_peered":"2024-04-06T14:04:02.225742+0000","last_clean":"2024-04-06T14:04:02.225742+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:04:02.225742+0000","last_undegraded":"2024-04-06T14:04:02.225742+0000","last_fullsized":"2024-04-06T14:04:02.225742+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513714+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:04:05.513714+0000","last_peered":"2024-04-06T14:04:05.513714+0000","last_clean":"2024-04-06T14:04:05.513714+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:04:05.513714+0000","last_undegraded":"2024-04-06T14:04:05.513714+0000","last_fullsized":"2024-04-06T14:04:05.513714+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225824+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:04:02.225824+0000","last_peered":"2024-04-06T14:04:02.225824+0000","last_clean":"2024-04-06T14:04:02.225824+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:04:02.225824+0000","last_undegraded":"2024-04-06T14:04:02.225824+0000","last_fullsized":"2024-04-06T14:04:02.225824+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"80'1","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.309374+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:04:03.309374+0000","last_peered":"2024-04-06T14:04:03.309374+0000","last_clean":"2024-04-06T14:04:03.309374+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:04:03.309374+0000","last_undegraded":"2024-04-06T14:04:03.309374+0000","last_fullsized":"2024-04-06T14:04:03.309374+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"81'1","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576464+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:04:02.576464+0000","last_peered":"2024-04-06T14:04:02.576464+0000","last_clean":"2024-04-06T14:04:02.576464+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:04:02.576464+0000","last_undegraded":"2024-04-06T14:04:02.576464+0000","last_fullsized":"2024-04-06T14:04:02.576464+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":14,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.467592+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:04:01.467592+0000","last_peered":"2024-04-06T14:04:01.467592+0000","last_clean":"2024-04-06T14:04:01.467592+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:04:01.467592+0000","last_undegraded":"2024-04-06T14:04:01.467592+0000","last_fullsized":"2024-04-06T14:04:01.467592+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225872+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:04:02.225872+0000","last_peered":"2024-04-06T14:04:02.225872+0000","last_clean":"2024-04-06T14:04:02.225872+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:04:02.225872+0000","last_undegraded":"2024-04-06T14:04:02.225872+0000","last_fullsized":"2024-04-06T14:04:02.225872+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489791+0000","last_change":"2024-04-06T14:04:01.454256+0000","last_active":"2024-04-06T14:04:01.489791+0000","last_peered":"2024-04-06T14:04:01.489791+0000","last_clean":"2024-04-06T14:04:01.489791+0000","last_became_active":"2024-04-06T14:04:01.452355+0000","last_became_peered":"2024-04-06T14:04:01.452355+0000","last_unstale":"2024-04-06T14:04:01.489791+0000","last_undegraded":"2024-04-06T14:04:01.489791+0000","last_fullsized":"2024-04-06T14:04:01.489791+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:34:42.131742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672108+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:04:01.672108+0000","last_peered":"2024-04-06T14:04:01.672108+0000","last_clean":"2024-04-06T14:04:01.672108+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:04:01.672108+0000","last_undegraded":"2024-04-06T14:04:01.672108+0000","last_fullsized":"2024-04-06T14:04:01.672108+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672134+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:04:01.672134+0000","last_peered":"2024-04-06T14:04:01.672134+0000","last_clean":"2024-04-06T14:04:01.672134+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:04:01.672134+0000","last_undegraded":"2024-04-06T14:04:01.672134+0000","last_fullsized":"2024-04-06T14:04:01.672134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574869+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:04:02.574869+0000","last_peered":"2024-04-06T14:04:02.574869+0000","last_clean":"2024-04-06T14:04:02.574869+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:04:02.574869+0000","last_undegraded":"2024-04-06T14:04:02.574869+0000","last_fullsized":"2024-04-06T14:04:02.574869+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575055+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:04:02.575055+0000","last_peered":"2024-04-06T14:04:02.575055+0000","last_clean":"2024-04-06T14:04:02.575055+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:04:02.575055+0000","last_undegraded":"2024-04-06T14:04:02.575055+0000","last_fullsized":"2024-04-06T14:04:02.575055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489833+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:04:01.489833+0000","last_peered":"2024-04-06T14:04:01.489833+0000","last_clean":"2024-04-06T14:04:01.489833+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:04:01.489833+0000","last_undegraded":"2024-04-06T14:04:01.489833+0000","last_fullsized":"2024-04-06T14:04:01.489833+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672268+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:04:01.672268+0000","last_peered":"2024-04-06T14:04:01.672268+0000","last_clean":"2024-04-06T14:04:01.672268+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:04:01.672268+0000","last_undegraded":"2024-04-06T14:04:01.672268+0000","last_fullsized":"2024-04-06T14:04:01.672268+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575443+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:04:02.575443+0000","last_peered":"2024-04-06T14:04:02.575443+0000","last_clean":"2024-04-06T14:04:02.575443+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:04:02.575443+0000","last_undegraded":"2024-04-06T14:04:02.575443+0000","last_fullsized":"2024-04-06T14:04:02.575443+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574980+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:04:02.574980+0000","last_peered":"2024-04-06T14:04:02.574980+0000","last_clean":"2024-04-06T14:04:02.574980+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:04:02.574980+0000","last_undegraded":"2024-04-06T14:04:02.574980+0000","last_fullsized":"2024-04-06T14:04:02.574980+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574778+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:04:02.574778+0000","last_peered":"2024-04-06T14:04:02.574778+0000","last_clean":"2024-04-06T14:04:02.574778+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:04:02.574778+0000","last_undegraded":"2024-04-06T14:04:02.574778+0000","last_fullsized":"2024-04-06T14:04:02.574778+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575547+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:04:02.575547+0000","last_peered":"2024-04-06T14:04:02.575547+0000","last_clean":"2024-04-06T14:04:02.575547+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:04:02.575547+0000","last_undegraded":"2024-04-06T14:04:02.575547+0000","last_fullsized":"2024-04-06T14:04:02.575547+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575324+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:04:02.575324+0000","last_peered":"2024-04-06T14:04:02.575324+0000","last_clean":"2024-04-06T14:04:02.575324+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:04:02.575324+0000","last_undegraded":"2024-04-06T14:04:02.575324+0000","last_fullsized":"2024-04-06T14:04:02.575324+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489822+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:04:01.489822+0000","last_peered":"2024-04-06T14:04:01.489822+0000","last_clean":"2024-04-06T14:04:01.489822+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:04:01.489822+0000","last_undegraded":"2024-04-06T14:04:01.489822+0000","last_fullsized":"2024-04-06T14:04:01.489822+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":376832,"data_stored":4132576,"data_compressed":246766,"data_compressed_allocated":315392,"data_compressed_original":4096000,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":64907,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":249,"num_read_kb":195,"num_write":150,"num_write_kb":106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":405504,"data_stored":194721,"data_compressed":5499,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":0,"internal_metadata":0},"log_size":132,"ondisk_log_size":132,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514515,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":868352,"data_stored":417921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.39600000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.745,"15min":0.745},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.622}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70299999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.68999999999999995}]},{"osd":10,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.66400000000000003}]}]},{"osd":10,"up_from":68,"seq":292057776150,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28004,"kb_used_data":988,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717148,"statfs":{"total":95995035648,"available":95966359552,"internally_reserved":0,"allocated":1011712,"data_stored":1070416,"data_compressed":37553,"data_compressed_allocated":49152,"data_compressed_original":598016,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.623}]},{"osd":2,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.497}]},{"osd":5,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.64900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.68999999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.66000000000000003}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.71599999999999997}]}]},{"osd":9,"up_from":63,"seq":270582939674,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27880,"kb_used_data":864,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717272,"statfs":{"total":95995035648,"available":95966486528,"internally_reserved":0,"allocated":884736,"data_stored":418157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.496}]},{"osd":6,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.38},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.71199999999999997}]},{"osd":8,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.58899999999999997}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.72099999999999997}]}]},{"osd":8,"up_from":56,"seq":240518168605,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":417746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54900000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.755,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76300000000000001,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.64700000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77700000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79900000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.623}]},{"osd":5,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74399999999999999,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.37},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80900000000000005,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.5,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71399999999999997,"15min":0.71599999999999997},"min":{"1min":0.504,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85899999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.56200000000000006,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.88,"5min":1.028,"15min":1.028},"last":0.64800000000000002}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.61299999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.58499999999999996}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":7,"up_from":49,"seq":210453397536,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32020,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713132,"statfs":{"total":95995035648,"available":95962247168,"internally_reserved":0,"allocated":929792,"data_stored":1009582,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.503},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.46600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.626,"5min":0.56599999999999995,"15min":0.55600000000000005},"min":{"1min":0.38500000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80800000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.36399999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54800000000000004,"15min":0.54300000000000004},"min":{"1min":0.44800000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72499999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.5,"15min":0.495},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.68100000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.57799999999999996},"min":{"1min":0.46400000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.61399999999999999,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.67300000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.624,"15min":0.61799999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.42099999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78700000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.72599999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.63},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.66300000000000003,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81899999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.75600000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.77700000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.79900000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71199999999999997}]}]},{"osd":6,"up_from":45,"seq":193273528356,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31964,"kb_used_data":852,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713188,"statfs":{"total":95995035648,"available":95962304512,"internally_reserved":0,"allocated":872448,"data_stored":417546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.39600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51000000000000001,"15min":0.502},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.311},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.44},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46899999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.433},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.621}]},{"osd":5,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83599999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.63600000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.65700000000000003,"15min":0.65500000000000003},"min":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.48399999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.495,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.79900000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.66100000000000003,"15min":0.65400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.65900000000000003}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51900000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.56100000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574902,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27924,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717228,"statfs":{"total":95995035648,"available":95966441472,"internally_reserved":0,"allocated":929792,"data_stored":478339,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.35899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.48799999999999999,"15min":0.47299999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Apr 6 14:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.51800000000000002,"15min":0.496},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75600000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52400000000000002,"15min":0.50700000000000001},"min":{"1min":0.39900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70399999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.50700000000000001,"15min":0.48699999999999999},"min":{"1min":0.35899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.52400000000000002,"15min":0.51000000000000001},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.76000000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46100000000000002,"15min":0.44700000000000001},"min":{"1min":0.38500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.41799999999999998},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79200000000000004,"5min":0.877,"15min":0.877},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56899999999999995,"15min":0.55700000000000005},"min":{"1min":0.48499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.79200000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.624},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.56100000000000005,"15min":0.54900000000000004},"min":{"1min":0.48399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80600000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.626,"15min":0.61599999999999999},"min":{"1min":0.38100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.875}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.70099999999999996,"15min":0.69499999999999995},"min":{"1min":0.53400000000000003,"5min":0.503,"15min":0.503},"max":{"1min":0.90700000000000003,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.65700000000000003,"15min":0.65000000000000002},"min":{"1min":0.56499999999999995,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.83799999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.82799999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.84299999999999997}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.84999999999999998}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27888,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717264,"statfs":{"total":95995035648,"available":95966478336,"internally_reserved":0,"allocated":892928,"data_stored":418817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51700000000000002},"min":{"1min":0.32700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.67200000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63800000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.50900000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.497,"15min":0.499},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65100000000000002,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60899999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.443,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46700000000000003,"15min":0.46899999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70799999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.502,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.65500000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.70199999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.504},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.627}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.67300000000000004}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.64900000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411378,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":954368,"data_stored":479641,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.47299999999999998,"15min":0.45300000000000001},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":1.9330000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45600000000000002,"15min":0.439},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53500000000000003,"15min":0.51700000000000002},"min":{"1min":0.438,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Apr 6 14:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53500000000000003,"15min":0.53200000000000003},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.74199999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.69899999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51900000000000002,"15min":0.51100000000000001},"min":{"1min":0.432,"5min":0.248,"15min":0.248},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.55200000000000005,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Apr 6 14:03:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.42999999999999999},"min":{"1min":0.32700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.307},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.42399999999999999,"15min":0.41499999999999998},"min":{"1min":0.31900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.34999999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60599999999999998,"15min":0.59999999999999998},"min":{"1min":0.44,"5min":0.372,"15min":0.372},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.52500000000000002}]},{"osd":9,"last update":"Sat Apr 6 14:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.67700000000000005}]},{"osd":10,"last update":"Sat Apr 6 14:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.61499999999999999}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72199999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149743,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36076,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93709076,"statfs":{"total":95995035648,"available":95958093824,"internally_reserved":0,"allocated":888832,"data_stored":417857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34100000000000003,"15min":0.33500000000000002},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.34300000000000003,"15min":0.33300000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.48499999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83399999999999996,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.505,"15min":0.48899999999999999},"min":{"1min":0.41499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71099999999999997,"5min":0.872,"15min":0.872},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53500000000000003,"15min":0.52500000000000002},"min":{"1min":0.41199999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.67100000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.79100000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65200000000000002,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.46700000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:07 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.49099999999999999,"15min":0.499},"min":{"1min":0.32300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60999999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:04:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.754,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46999999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.66600000000000004}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.42299999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.71399999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68600000000000005}]}]},{"osd":4,"up_from":33,"seq":141733920811,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27872,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717280,"statfs":{"total":95995035648,"available":95966494720,"internally_reserved":0,"allocated":876544,"data_stored":418328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35399999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.40200000000000002,"15min":0.378},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.70199999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":1.036,"15min":1.036},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.55800000000000005,"15min":0.53600000000000003},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.50700000000000001},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76500000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.628},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.47999999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68500000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.55600000000000005,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.48299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.81799999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.52400000000000002,"15min":0.50800000000000001},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.505}]},{"osd":7,"last update":"Sat Apr 6 14:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.44900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57699999999999996,"15min":0.56299999999999994},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.51700000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.68000000000000005,"15min":0.67700000000000005},"min":{"1min":0.52200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85799999999999998,"5min":0.879,"15min":0.879},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73899999999999999,"15min":0.746},"min":{"1min":0.57999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81999999999999995,"5min":1.694,"15min":1.694},"last":0.64200000000000002}]},{"osd":9,"last update":"Sat Apr 6 14:03:53 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.54500000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.57699999999999996}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68200000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32036,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713116,"statfs":{"total":95995035648,"available":95962230784,"internally_reserved":0,"allocated":946176,"data_stored":1009252,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.49299999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.48999999999999999,"15min":0.47699999999999998},"min":{"1min":0.36599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51100000000000001,"15min":0.497},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.441},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.499,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.38300000000000001,"15min":0.37},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.317,"5min":0.223,"15min":0.223},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.52900000000000003,"15min":0.51600000000000001},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.58799999999999997}]},{"osd":4,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57199999999999995,"15min":0.56200000000000006},"min":{"1min":0.45400000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83199999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.627,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.876,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.63,"15min":0.62},"min":{"1min":0.503,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.63100000000000001,"15min":0.62},"min":{"1min":0.52100000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.89700000000000002,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.52700000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.625,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.68400000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.88200000000000001,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.64000000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.61499999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.626}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":61000,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":62302,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":62923,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:04:47.244 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-06T14:04:47.467 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:47.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:47 smithi105 ceph-mon[46969]: pgmap v242: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:47.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:47 smithi090 ceph-mon[43420]: pgmap v242: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:47 smithi134 ceph-mon[47549]: pgmap v242: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 -- 172.21.15.90:0/808291211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 msgr2=0x7f9598107220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 --2- 172.21.15.90:0/808291211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f9598107220 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f9580009990 tx=0x7f9580009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 -- 172.21.15.90:0/808291211 shutdown_connections 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 --2- 172.21.15.90:0/808291211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95981017a0 0x7f959810dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 --2- 172.21.15.90:0/808291211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f9598107220 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 --2- 172.21.15.90:0/808291211 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9598100e00 0x7f9598101260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 -- 172.21.15.90:0/808291211 >> 172.21.15.90:0/808291211 conn(0x7f9598077fa0 msgr2=0x7f95980783b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:48.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 -- 172.21.15.90:0/808291211 shutdown_connections 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.460+0000 7f959cdc1700 1 -- 172.21.15.90:0/808291211 wait complete. 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.461+0000 7f959cdc1700 1 Processor -- start 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.461+0000 7f959cdc1700 1 -- start start 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.461+0000 7f959cdc1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 0x7f9598197230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.461+0000 7f959cdc1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f95981017a0 0x7f9598197770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.461+0000 7f959cdc1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959cdc1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f959819c130 con 0x7f9598106e40 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959cdc1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f959819c2a0 con 0x7f9598100e00 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959cdc1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f959819c460 con 0x7f95981017a0 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51932/0 (socket says 172.21.15.90:51932) 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 -- 172.21.15.90:0/1135523922 learned_addr learned my addr 172.21.15.90:0/1135523922 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:48.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9595d9b700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f95981017a0 0x7f9598197770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959659c700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 0x7f9598197230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 -- 172.21.15.90:0/1135523922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f95981017a0 msgr2=0x7f9598197770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f95981017a0 0x7f9598197770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 -- 172.21.15.90:0/1135523922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 msgr2=0x7f9598197230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 0x7f9598197230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 -- 172.21.15.90:0/1135523922 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95800095f0 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959659c700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 0x7f9598197230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f9596d9d700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f959000cca0 tx=0x7f959000a060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f959000aba0 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f959000ad00 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f959819c740 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95900115e0 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.462+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95981a3fa0 con 0x7f9598106e40 2024-04-06T14:04:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.463+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9598109360 con 0x7f9598106e40 2024-04-06T14:04:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.464+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9590012420 con 0x7f9598106e40 2024-04-06T14:04:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.465+0000 7f958f7fe700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 0x7f95840a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.465+0000 7f959659c700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 0x7f95840a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.465+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f95900cc9c0 con 0x7f9598106e40 2024-04-06T14:04:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.465+0000 7f959659c700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 0x7f95840a3670 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f9580006010 tx=0x7f958001a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.470+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f95900962b0 con 0x7f9598106e40 2024-04-06T14:04:48.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:48 smithi105 ceph-mon[46969]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:48.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:48 smithi090 ceph-mon[43420]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:48.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:48 smithi134 ceph-mon[47549]: from='client.14967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:48.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.659+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f959819c930 con 0x7f95840a11b0 2024-04-06T14:04:48.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.669+0000 7f958f7fe700 1 -- 172.21.15.90:0/1135523922 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+341838 (secure 0 0 0) 0x7f959819c930 con 0x7f95840a11b0 2024-04-06T14:04:48.674 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 msgr2=0x7f95840a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 0x7f95840a3670 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f9580006010 tx=0x7f958001a040 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 msgr2=0x7f959819bb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f959000cca0 tx=0x7f959000a060 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 shutdown_connections 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f95840a11b0 0x7f95840a3670 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9598106e40 0x7f959819bb20 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9598100e00 0x7f9598197230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 --2- 172.21.15.90:0/1135523922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f95981017a0 0x7f9598197770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:48.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 >> 172.21.15.90:0/1135523922 conn(0x7f9598077fa0 msgr2=0x7f95980ffba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:48.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 shutdown_connections 2024-04-06T14:04:48.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:48.672+0000 7f959cdc1700 1 -- 172.21.15.90:0/1135523922 wait complete. 2024-04-06T14:04:48.678 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:04:49.090 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":243,"stamp":"2024-04-06T14:04:48.169869+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655509,"num_objects":64,"num_object_clones":0,"num_object_copies":192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":64,"num_whiteouts":0,"num_read":390,"num_read_kb":314,"num_write":333,"num_write_kb":3046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":271,"ondisk_log_size":271,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":359448,"kb_used_data":10680,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124582376,"statfs":{"total":1151940427776,"available":1151572353024,"internally_reserved":0,"allocated":10936320,"data_stored":6973602,"data_compressed":112659,"data_compressed_allocated":147456,"data_compressed_original":1794048,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1652,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":66,"num_read_kb":32,"num_write":1,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002676"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489854+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:04:01.489854+0000","last_peered":"2024-04-06T14:04:01.489854+0000","last_clean":"2024-04-06T14:04:01.489854+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:04:01.489854+0000","last_undegraded":"2024-04-06T14:04:01.489854+0000","last_fullsized":"2024-04-06T14:04:01.489854+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574988+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:04:02.574988+0000","last_peered":"2024-04-06T14:04:02.574988+0000","last_clean":"2024-04-06T14:04:02.574988+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:04:02.574988+0000","last_undegraded":"2024-04-06T14:04:02.574988+0000","last_fullsized":"2024-04-06T14:04:02.574988+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574769+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:04:02.574769+0000","last_peered":"2024-04-06T14:04:02.574769+0000","last_clean":"2024-04-06T14:04:02.574769+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:04:02.574769+0000","last_undegraded":"2024-04-06T14:04:02.574769+0000","last_fullsized":"2024-04-06T14:04:02.574769+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470214+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:04:01.470214+0000","last_peered":"2024-04-06T14:04:01.470214+0000","last_clean":"2024-04-06T14:04:01.470214+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:04:01.470214+0000","last_undegraded":"2024-04-06T14:04:01.470214+0000","last_fullsized":"2024-04-06T14:04:01.470214+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575085+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:04:02.575085+0000","last_peered":"2024-04-06T14:04:02.575085+0000","last_clean":"2024-04-06T14:04:02.575085+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:04:02.575085+0000","last_undegraded":"2024-04-06T14:04:02.575085+0000","last_fullsized":"2024-04-06T14:04:02.575085+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436129+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:04:02.436129+0000","last_peered":"2024-04-06T14:04:02.436129+0000","last_clean":"2024-04-06T14:04:02.436129+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:04:02.436129+0000","last_undegraded":"2024-04-06T14:04:02.436129+0000","last_fullsized":"2024-04-06T14:04:02.436129+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225702+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:04:02.225702+0000","last_peered":"2024-04-06T14:04:02.225702+0000","last_clean":"2024-04-06T14:04:02.225702+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:04:02.225702+0000","last_undegraded":"2024-04-06T14:04:02.225702+0000","last_fullsized":"2024-04-06T14:04:02.225702+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575535+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:04:02.575535+0000","last_peered":"2024-04-06T14:04:02.575535+0000","last_clean":"2024-04-06T14:04:02.575535+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:04:02.575535+0000","last_undegraded":"2024-04-06T14:04:02.575535+0000","last_fullsized":"2024-04-06T14:04:02.575535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575974+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:04:02.575974+0000","last_peered":"2024-04-06T14:04:02.575974+0000","last_clean":"2024-04-06T14:04:02.575974+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:04:02.575974+0000","last_undegraded":"2024-04-06T14:04:02.575974+0000","last_fullsized":"2024-04-06T14:04:02.575974+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436412+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:04:02.436412+0000","last_peered":"2024-04-06T14:04:02.436412+0000","last_clean":"2024-04-06T14:04:02.436412+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:04:02.436412+0000","last_undegraded":"2024-04-06T14:04:02.436412+0000","last_fullsized":"2024-04-06T14:04:02.436412+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225803+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:04:02.225803+0000","last_peered":"2024-04-06T14:04:02.225803+0000","last_clean":"2024-04-06T14:04:02.225803+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:04:02.225803+0000","last_undegraded":"2024-04-06T14:04:02.225803+0000","last_fullsized":"2024-04-06T14:04:02.225803+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.470202+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:04:01.470202+0000","last_peered":"2024-04-06T14:04:01.470202+0000","last_clean":"2024-04-06T14:04:01.470202+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:04:01.470202+0000","last_undegraded":"2024-04-06T14:04:01.470202+0000","last_fullsized":"2024-04-06T14:04:01.470202+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575159+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:04:02.575159+0000","last_peered":"2024-04-06T14:04:02.575159+0000","last_clean":"2024-04-06T14:04:02.575159+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:04:02.575159+0000","last_undegraded":"2024-04-06T14:04:02.575159+0000","last_fullsized":"2024-04-06T14:04:02.575159+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574684+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:04:02.574684+0000","last_peered":"2024-04-06T14:04:02.574684+0000","last_clean":"2024-04-06T14:04:02.574684+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:04:02.574684+0000","last_undegraded":"2024-04-06T14:04:02.574684+0000","last_fullsized":"2024-04-06T14:04:02.574684+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574459+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:04:02.574459+0000","last_peered":"2024-04-06T14:04:02.574459+0000","last_clean":"2024-04-06T14:04:02.574459+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:04:02.574459+0000","last_undegraded":"2024-04-06T14:04:02.574459+0000","last_fullsized":"2024-04-06T14:04:02.574459+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575094+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:04:02.575094+0000","last_peered":"2024-04-06T14:04:02.575094+0000","last_clean":"2024-04-06T14:04:02.575094+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:04:02.575094+0000","last_undegraded":"2024-04-06T14:04:02.575094+0000","last_fullsized":"2024-04-06T14:04:02.575094+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489865+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:04:01.489865+0000","last_peered":"2024-04-06T14:04:01.489865+0000","last_clean":"2024-04-06T14:04:01.489865+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:04:01.489865+0000","last_undegraded":"2024-04-06T14:04:01.489865+0000","last_fullsized":"2024-04-06T14:04:01.489865+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437817+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:04:02.437817+0000","last_peered":"2024-04-06T14:04:02.437817+0000","last_clean":"2024-04-06T14:04:02.437817+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:04:02.437817+0000","last_undegraded":"2024-04-06T14:04:02.437817+0000","last_fullsized":"2024-04-06T14:04:02.437817+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672247+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:04:01.672247+0000","last_peered":"2024-04-06T14:04:01.672247+0000","last_clean":"2024-04-06T14:04:01.672247+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:04:01.672247+0000","last_undegraded":"2024-04-06T14:04:01.672247+0000","last_fullsized":"2024-04-06T14:04:01.672247+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":22,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513730+0000","last_change":"2024-04-06T14:04:01.028407+0000","last_active":"2024-04-06T14:04:05.513730+0000","last_peered":"2024-04-06T14:04:05.513730+0000","last_clean":"2024-04-06T14:04:05.513730+0000","last_became_active":"2024-04-06T14:04:01.028036+0000","last_became_peered":"2024-04-06T14:04:01.028036+0000","last_unstale":"2024-04-06T14:04:05.513730+0000","last_undegraded":"2024-04-06T14:04:05.513730+0000","last_fullsized":"2024-04-06T14:04:05.513730+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:08:53.486282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575627+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:04:02.575627+0000","last_peered":"2024-04-06T14:04:02.575627+0000","last_clean":"2024-04-06T14:04:02.575627+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:04:02.575627+0000","last_undegraded":"2024-04-06T14:04:02.575627+0000","last_fullsized":"2024-04-06T14:04:02.575627+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.115761+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:04:04.115761+0000","last_peered":"2024-04-06T14:04:04.115761+0000","last_clean":"2024-04-06T14:04:04.115761+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:04:04.115761+0000","last_undegraded":"2024-04-06T14:04:04.115761+0000","last_fullsized":"2024-04-06T14:04:04.115761+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575409+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:04:02.575409+0000","last_peered":"2024-04-06T14:04:02.575409+0000","last_clean":"2024-04-06T14:04:02.575409+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:04:02.575409+0000","last_undegraded":"2024-04-06T14:04:02.575409+0000","last_fullsized":"2024-04-06T14:04:02.575409+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512073+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:04:05.512073+0000","last_peered":"2024-04-06T14:04:05.512073+0000","last_clean":"2024-04-06T14:04:05.512073+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:04:05.512073+0000","last_undegraded":"2024-04-06T14:04:05.512073+0000","last_fullsized":"2024-04-06T14:04:05.512073+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575221+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:04:02.575221+0000","last_peered":"2024-04-06T14:04:02.575221+0000","last_clean":"2024-04-06T14:04:02.575221+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:04:02.575221+0000","last_undegraded":"2024-04-06T14:04:02.575221+0000","last_fullsized":"2024-04-06T14:04:02.575221+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575477+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:04:02.575477+0000","last_peered":"2024-04-06T14:04:02.575477+0000","last_clean":"2024-04-06T14:04:02.575477+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:04:02.575477+0000","last_undegraded":"2024-04-06T14:04:02.575477+0000","last_fullsized":"2024-04-06T14:04:02.575477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574918+0000","last_change":"2024-04-06T14:04:01.457311+0000","last_active":"2024-04-06T14:04:02.574918+0000","last_peered":"2024-04-06T14:04:02.574918+0000","last_clean":"2024-04-06T14:04:02.574918+0000","last_became_active":"2024-04-06T14:04:01.456954+0000","last_became_peered":"2024-04-06T14:04:01.456954+0000","last_unstale":"2024-04-06T14:04:02.574918+0000","last_undegraded":"2024-04-06T14:04:02.574918+0000","last_fullsized":"2024-04-06T14:04:02.574918+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:22:59.524995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"81'5","reported_seq":56,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:41.568815+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:04:41.568815+0000","last_peered":"2024-04-06T14:04:41.568815+0000","last_clean":"2024-04-06T14:04:41.568815+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:04:41.568815+0000","last_undegraded":"2024-04-06T14:04:41.568815+0000","last_fullsized":"2024-04-06T14:04:41.568815+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575517+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:04:02.575517+0000","last_peered":"2024-04-06T14:04:02.575517+0000","last_clean":"2024-04-06T14:04:02.575517+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:04:02.575517+0000","last_undegraded":"2024-04-06T14:04:02.575517+0000","last_fullsized":"2024-04-06T14:04:02.575517+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512715+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:04:05.512715+0000","last_peered":"2024-04-06T14:04:05.512715+0000","last_clean":"2024-04-06T14:04:05.512715+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:04:05.512715+0000","last_undegraded":"2024-04-06T14:04:05.512715+0000","last_fullsized":"2024-04-06T14:04:05.512715+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574780+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:04:02.574780+0000","last_peered":"2024-04-06T14:04:02.574780+0000","last_clean":"2024-04-06T14:04:02.574780+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:04:02.574780+0000","last_undegraded":"2024-04-06T14:04:02.574780+0000","last_fullsized":"2024-04-06T14:04:02.574780+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.514049+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:04:05.514049+0000","last_peered":"2024-04-06T14:04:05.514049+0000","last_clean":"2024-04-06T14:04:05.514049+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:04:05.514049+0000","last_undegraded":"2024-04-06T14:04:05.514049+0000","last_fullsized":"2024-04-06T14:04:05.514049+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575621+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:04:02.575621+0000","last_peered":"2024-04-06T14:04:02.575621+0000","last_clean":"2024-04-06T14:04:02.575621+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:04:02.575621+0000","last_undegraded":"2024-04-06T14:04:02.575621+0000","last_fullsized":"2024-04-06T14:04:02.575621+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.309221+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:04:03.309221+0000","last_peered":"2024-04-06T14:04:03.309221+0000","last_clean":"2024-04-06T14:04:03.309221+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:04:03.309221+0000","last_undegraded":"2024-04-06T14:04:03.309221+0000","last_fullsized":"2024-04-06T14:04:03.309221+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.437700+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:04:02.437700+0000","last_peered":"2024-04-06T14:04:02.437700+0000","last_clean":"2024-04-06T14:04:02.437700+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:04:02.437700+0000","last_undegraded":"2024-04-06T14:04:02.437700+0000","last_fullsized":"2024-04-06T14:04:02.437700+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"81'3","reported_seq":52,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:41.568153+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:04:41.568153+0000","last_peered":"2024-04-06T14:04:41.568153+0000","last_clean":"2024-04-06T14:04:41.568153+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:04:41.568153+0000","last_undegraded":"2024-04-06T14:04:41.568153+0000","last_fullsized":"2024-04-06T14:04:41.568153+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":37,"num_read_kb":37,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575670+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:04:02.575670+0000","last_peered":"2024-04-06T14:04:02.575670+0000","last_clean":"2024-04-06T14:04:02.575670+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:04:02.575670+0000","last_undegraded":"2024-04-06T14:04:02.575670+0000","last_fullsized":"2024-04-06T14:04:02.575670+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507363+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:04:04.507363+0000","last_peered":"2024-04-06T14:04:04.507363+0000","last_clean":"2024-04-06T14:04:04.507363+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:04:04.507363+0000","last_undegraded":"2024-04-06T14:04:04.507363+0000","last_fullsized":"2024-04-06T14:04:04.507363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575323+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:04:02.575323+0000","last_peered":"2024-04-06T14:04:02.575323+0000","last_clean":"2024-04-06T14:04:02.575323+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:04:02.575323+0000","last_undegraded":"2024-04-06T14:04:02.575323+0000","last_fullsized":"2024-04-06T14:04:02.575323+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"81'4","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.504954+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:04:04.504954+0000","last_peered":"2024-04-06T14:04:04.504954+0000","last_clean":"2024-04-06T14:04:04.504954+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:04:04.504954+0000","last_undegraded":"2024-04-06T14:04:04.504954+0000","last_fullsized":"2024-04-06T14:04:04.504954+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575352+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:04:02.575352+0000","last_peered":"2024-04-06T14:04:02.575352+0000","last_clean":"2024-04-06T14:04:02.575352+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:04:02.575352+0000","last_undegraded":"2024-04-06T14:04:02.575352+0000","last_fullsized":"2024-04-06T14:04:02.575352+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"81'28","reported_seq":122,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:42.580477+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:04:42.580477+0000","last_peered":"2024-04-06T14:04:42.580477+0000","last_clean":"2024-04-06T14:04:42.580477+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:04:42.580477+0000","last_undegraded":"2024-04-06T14:04:42.580477+0000","last_fullsized":"2024-04-06T14:04:42.580477+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61000,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":82,"num_read_kb":100,"num_write":29,"num_write_kb":69,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575168+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:04:02.575168+0000","last_peered":"2024-04-06T14:04:02.575168+0000","last_clean":"2024-04-06T14:04:02.575168+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:04:02.575168+0000","last_undegraded":"2024-04-06T14:04:02.575168+0000","last_fullsized":"2024-04-06T14:04:02.575168+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":19,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513503+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:04:05.513503+0000","last_peered":"2024-04-06T14:04:05.513503+0000","last_clean":"2024-04-06T14:04:05.513503+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:04:05.513503+0000","last_undegraded":"2024-04-06T14:04:05.513503+0000","last_fullsized":"2024-04-06T14:04:05.513503+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574857+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:04:02.574857+0000","last_peered":"2024-04-06T14:04:02.574857+0000","last_clean":"2024-04-06T14:04:02.574857+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:04:02.574857+0000","last_undegraded":"2024-04-06T14:04:02.574857+0000","last_fullsized":"2024-04-06T14:04:02.574857+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.506929+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:04:04.506929+0000","last_peered":"2024-04-06T14:04:04.506929+0000","last_clean":"2024-04-06T14:04:04.506929+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:04:04.506929+0000","last_undegraded":"2024-04-06T14:04:04.506929+0000","last_fullsized":"2024-04-06T14:04:04.506929+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225762+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:04:02.225762+0000","last_peered":"2024-04-06T14:04:02.225762+0000","last_clean":"2024-04-06T14:04:02.225762+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:04:02.225762+0000","last_undegraded":"2024-04-06T14:04:02.225762+0000","last_fullsized":"2024-04-06T14:04:02.225762+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"81'4","reported_seq":59,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:44.575053+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:04:44.575053+0000","last_peered":"2024-04-06T14:04:44.575053+0000","last_clean":"2024-04-06T14:04:44.575053+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:04:44.575053+0000","last_undegraded":"2024-04-06T14:04:44.575053+0000","last_fullsized":"2024-04-06T14:04:44.575053+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":933,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":4,"num_write":3,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225612+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:04:02.225612+0000","last_peered":"2024-04-06T14:04:02.225612+0000","last_clean":"2024-04-06T14:04:02.225612+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:04:02.225612+0000","last_undegraded":"2024-04-06T14:04:02.225612+0000","last_fullsized":"2024-04-06T14:04:02.225612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"81'15","reported_seq":28,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513774+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:04:05.513774+0000","last_peered":"2024-04-06T14:04:05.513774+0000","last_clean":"2024-04-06T14:04:05.513774+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:04:05.513774+0000","last_undegraded":"2024-04-06T14:04:05.513774+0000","last_fullsized":"2024-04-06T14:04:05.513774+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574613+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:04:02.574613+0000","last_peered":"2024-04-06T14:04:02.574613+0000","last_clean":"2024-04-06T14:04:02.574613+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:04:02.574613+0000","last_undegraded":"2024-04-06T14:04:02.574613+0000","last_fullsized":"2024-04-06T14:04:02.574613+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'2","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:06.516280+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:04:06.516280+0000","last_peered":"2024-04-06T14:04:06.516280+0000","last_clean":"2024-04-06T14:04:06.516280+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:04:06.516280+0000","last_undegraded":"2024-04-06T14:04:06.516280+0000","last_fullsized":"2024-04-06T14:04:06.516280+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489844+0000","last_change":"2024-04-06T14:04:01.455152+0000","last_active":"2024-04-06T14:04:01.489844+0000","last_peered":"2024-04-06T14:04:01.489844+0000","last_clean":"2024-04-06T14:04:01.489844+0000","last_became_active":"2024-04-06T14:04:01.454827+0000","last_became_peered":"2024-04-06T14:04:01.454827+0000","last_unstale":"2024-04-06T14:04:01.489844+0000","last_undegraded":"2024-04-06T14:04:01.489844+0000","last_fullsized":"2024-04-06T14:04:01.489844+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:55:26.359159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.438338+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:04:01.438338+0000","last_peered":"2024-04-06T14:04:01.438338+0000","last_clean":"2024-04-06T14:04:01.438338+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:04:01.438338+0000","last_undegraded":"2024-04-06T14:04:01.438338+0000","last_fullsized":"2024-04-06T14:04:01.438338+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"81'1","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.200634+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:04:03.200634+0000","last_peered":"2024-04-06T14:04:03.200634+0000","last_clean":"2024-04-06T14:04:03.200634+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:04:03.200634+0000","last_undegraded":"2024-04-06T14:04:03.200634+0000","last_fullsized":"2024-04-06T14:04:03.200634+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576581+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:04:02.576581+0000","last_peered":"2024-04-06T14:04:02.576581+0000","last_clean":"2024-04-06T14:04:02.576581+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:04:02.576581+0000","last_undegraded":"2024-04-06T14:04:02.576581+0000","last_fullsized":"2024-04-06T14:04:02.576581+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.436536+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:04:02.436536+0000","last_peered":"2024-04-06T14:04:02.436536+0000","last_clean":"2024-04-06T14:04:02.436536+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:04:02.436536+0000","last_undegraded":"2024-04-06T14:04:02.436536+0000","last_fullsized":"2024-04-06T14:04:02.436536+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575047+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:04:02.575047+0000","last_peered":"2024-04-06T14:04:02.575047+0000","last_clean":"2024-04-06T14:04:02.575047+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:04:02.575047+0000","last_undegraded":"2024-04-06T14:04:02.575047+0000","last_fullsized":"2024-04-06T14:04:02.575047+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"81'4","reported_seq":55,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:43.572538+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:04:43.572538+0000","last_peered":"2024-04-06T14:04:43.572538+0000","last_clean":"2024-04-06T14:04:43.572538+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:04:43.572538+0000","last_undegraded":"2024-04-06T14:04:43.572538+0000","last_fullsized":"2024-04-06T14:04:43.572538+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":39,"num_read_kb":39,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575189+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:04:02.575189+0000","last_peered":"2024-04-06T14:04:02.575189+0000","last_clean":"2024-04-06T14:04:02.575189+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:04:02.575189+0000","last_undegraded":"2024-04-06T14:04:02.575189+0000","last_fullsized":"2024-04-06T14:04:02.575189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:06.515167+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:04:06.515167+0000","last_peered":"2024-04-06T14:04:06.515167+0000","last_clean":"2024-04-06T14:04:06.515167+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:04:06.515167+0000","last_undegraded":"2024-04-06T14:04:06.515167+0000","last_fullsized":"2024-04-06T14:04:06.515167+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":334,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575511+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:04:02.575511+0000","last_peered":"2024-04-06T14:04:02.575511+0000","last_clean":"2024-04-06T14:04:02.575511+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:04:02.575511+0000","last_undegraded":"2024-04-06T14:04:02.575511+0000","last_fullsized":"2024-04-06T14:04:02.575511+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574458+0000","last_change":"2024-04-06T14:04:01.457132+0000","last_active":"2024-04-06T14:04:02.574458+0000","last_peered":"2024-04-06T14:04:02.574458+0000","last_clean":"2024-04-06T14:04:02.574458+0000","last_became_active":"2024-04-06T14:04:01.456782+0000","last_became_peered":"2024-04-06T14:04:01.456782+0000","last_unstale":"2024-04-06T14:04:02.574458+0000","last_undegraded":"2024-04-06T14:04:02.574458+0000","last_fullsized":"2024-04-06T14:04:02.574458+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:34:37.759113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.512264+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:04:05.512264+0000","last_peered":"2024-04-06T14:04:05.512264+0000","last_clean":"2024-04-06T14:04:05.512264+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:04:05.512264+0000","last_undegraded":"2024-04-06T14:04:05.512264+0000","last_fullsized":"2024-04-06T14:04:05.512264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"81'1","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.505412+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:04:04.505412+0000","last_peered":"2024-04-06T14:04:04.505412+0000","last_clean":"2024-04-06T14:04:04.505412+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:04:04.505412+0000","last_undegraded":"2024-04-06T14:04:04.505412+0000","last_fullsized":"2024-04-06T14:04:04.505412+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574508+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:04:02.574508+0000","last_peered":"2024-04-06T14:04:02.574508+0000","last_clean":"2024-04-06T14:04:02.574508+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:04:02.574508+0000","last_undegraded":"2024-04-06T14:04:02.574508+0000","last_fullsized":"2024-04-06T14:04:02.574508+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"81'5","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:10.432735+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:04:10.432735+0000","last_peered":"2024-04-06T14:04:10.432735+0000","last_clean":"2024-04-06T14:04:10.432735+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:04:10.432735+0000","last_undegraded":"2024-04-06T14:04:10.432735+0000","last_fullsized":"2024-04-06T14:04:10.432735+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575544+0000","last_change":"2024-04-06T14:04:01.461874+0000","last_active":"2024-04-06T14:04:02.575544+0000","last_peered":"2024-04-06T14:04:02.575544+0000","last_clean":"2024-04-06T14:04:02.575544+0000","last_became_active":"2024-04-06T14:04:01.461652+0000","last_became_peered":"2024-04-06T14:04:01.461652+0000","last_unstale":"2024-04-06T14:04:02.575544+0000","last_undegraded":"2024-04-06T14:04:02.575544+0000","last_fullsized":"2024-04-06T14:04:02.575544+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:01:58.862215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507264+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:04:04.507264+0000","last_peered":"2024-04-06T14:04:04.507264+0000","last_clean":"2024-04-06T14:04:04.507264+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:04:04.507264+0000","last_undegraded":"2024-04-06T14:04:04.507264+0000","last_fullsized":"2024-04-06T14:04:04.507264+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575075+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:04:02.575075+0000","last_peered":"2024-04-06T14:04:02.575075+0000","last_clean":"2024-04-06T14:04:02.575075+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:04:02.575075+0000","last_undegraded":"2024-04-06T14:04:02.575075+0000","last_fullsized":"2024-04-06T14:04:02.575075+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.507554+0000","last_change":"2024-04-06T14:04:01.035529+0000","last_active":"2024-04-06T14:04:04.507554+0000","last_peered":"2024-04-06T14:04:04.507554+0000","last_clean":"2024-04-06T14:04:04.507554+0000","last_became_active":"2024-04-06T14:04:01.035205+0000","last_became_peered":"2024-04-06T14:04:01.035205+0000","last_unstale":"2024-04-06T14:04:04.507554+0000","last_undegraded":"2024-04-06T14:04:04.507554+0000","last_fullsized":"2024-04-06T14:04:04.507554+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:59:17.605780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574775+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:04:02.574775+0000","last_peered":"2024-04-06T14:04:02.574775+0000","last_clean":"2024-04-06T14:04:02.574775+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:04:02.574775+0000","last_undegraded":"2024-04-06T14:04:02.574775+0000","last_fullsized":"2024-04-06T14:04:02.574775+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"81'3","reported_seq":15,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.506884+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:04:04.506884+0000","last_peered":"2024-04-06T14:04:04.506884+0000","last_clean":"2024-04-06T14:04:04.506884+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:04:04.506884+0000","last_undegraded":"2024-04-06T14:04:04.506884+0000","last_fullsized":"2024-04-06T14:04:04.506884+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489773+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:04:01.489773+0000","last_peered":"2024-04-06T14:04:01.489773+0000","last_clean":"2024-04-06T14:04:01.489773+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:04:01.489773+0000","last_undegraded":"2024-04-06T14:04:01.489773+0000","last_fullsized":"2024-04-06T14:04:01.489773+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"81'1","reported_seq":13,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.560630+0000","last_change":"2024-04-06T14:04:01.032612+0000","last_active":"2024-04-06T14:04:02.560630+0000","last_peered":"2024-04-06T14:04:02.560630+0000","last_clean":"2024-04-06T14:04:02.560630+0000","last_became_active":"2024-04-06T14:04:01.032115+0000","last_became_peered":"2024-04-06T14:04:01.032115+0000","last_unstale":"2024-04-06T14:04:02.560630+0000","last_undegraded":"2024-04-06T14:04:02.560630+0000","last_fullsized":"2024-04-06T14:04:02.560630+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:40.607887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575432+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:04:02.575432+0000","last_peered":"2024-04-06T14:04:02.575432+0000","last_clean":"2024-04-06T14:04:02.575432+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:04:02.575432+0000","last_undegraded":"2024-04-06T14:04:02.575432+0000","last_fullsized":"2024-04-06T14:04:02.575432+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"81'4","reported_seq":17,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:04.508618+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:04:04.508618+0000","last_peered":"2024-04-06T14:04:04.508618+0000","last_clean":"2024-04-06T14:04:04.508618+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:04:04.508618+0000","last_undegraded":"2024-04-06T14:04:04.508618+0000","last_fullsized":"2024-04-06T14:04:04.508618+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225742+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:04:02.225742+0000","last_peered":"2024-04-06T14:04:02.225742+0000","last_clean":"2024-04-06T14:04:02.225742+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:04:02.225742+0000","last_undegraded":"2024-04-06T14:04:02.225742+0000","last_fullsized":"2024-04-06T14:04:02.225742+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":16,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:05.513714+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:04:05.513714+0000","last_peered":"2024-04-06T14:04:05.513714+0000","last_clean":"2024-04-06T14:04:05.513714+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:04:05.513714+0000","last_undegraded":"2024-04-06T14:04:05.513714+0000","last_fullsized":"2024-04-06T14:04:05.513714+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225824+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:04:02.225824+0000","last_peered":"2024-04-06T14:04:02.225824+0000","last_clean":"2024-04-06T14:04:02.225824+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:04:02.225824+0000","last_undegraded":"2024-04-06T14:04:02.225824+0000","last_fullsized":"2024-04-06T14:04:02.225824+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"80'1","reported_seq":14,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:03.309374+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:04:03.309374+0000","last_peered":"2024-04-06T14:04:03.309374+0000","last_clean":"2024-04-06T14:04:03.309374+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:04:03.309374+0000","last_undegraded":"2024-04-06T14:04:03.309374+0000","last_fullsized":"2024-04-06T14:04:03.309374+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"81'1","reported_seq":12,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.576464+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:04:02.576464+0000","last_peered":"2024-04-06T14:04:02.576464+0000","last_clean":"2024-04-06T14:04:02.576464+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:04:02.576464+0000","last_undegraded":"2024-04-06T14:04:02.576464+0000","last_fullsized":"2024-04-06T14:04:02.576464+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":14,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.467592+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:04:01.467592+0000","last_peered":"2024-04-06T14:04:01.467592+0000","last_clean":"2024-04-06T14:04:01.467592+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:04:01.467592+0000","last_undegraded":"2024-04-06T14:04:01.467592+0000","last_fullsized":"2024-04-06T14:04:01.467592+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.225872+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:04:02.225872+0000","last_peered":"2024-04-06T14:04:02.225872+0000","last_clean":"2024-04-06T14:04:02.225872+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:04:02.225872+0000","last_undegraded":"2024-04-06T14:04:02.225872+0000","last_fullsized":"2024-04-06T14:04:02.225872+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489791+0000","last_change":"2024-04-06T14:04:01.454256+0000","last_active":"2024-04-06T14:04:01.489791+0000","last_peered":"2024-04-06T14:04:01.489791+0000","last_clean":"2024-04-06T14:04:01.489791+0000","last_became_active":"2024-04-06T14:04:01.452355+0000","last_became_peered":"2024-04-06T14:04:01.452355+0000","last_unstale":"2024-04-06T14:04:01.489791+0000","last_undegraded":"2024-04-06T14:04:01.489791+0000","last_fullsized":"2024-04-06T14:04:01.489791+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:34:42.131742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672108+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:04:01.672108+0000","last_peered":"2024-04-06T14:04:01.672108+0000","last_clean":"2024-04-06T14:04:01.672108+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:04:01.672108+0000","last_undegraded":"2024-04-06T14:04:01.672108+0000","last_fullsized":"2024-04-06T14:04:01.672108+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672134+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:04:01.672134+0000","last_peered":"2024-04-06T14:04:01.672134+0000","last_clean":"2024-04-06T14:04:01.672134+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:04:01.672134+0000","last_undegraded":"2024-04-06T14:04:01.672134+0000","last_fullsized":"2024-04-06T14:04:01.672134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574869+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:04:02.574869+0000","last_peered":"2024-04-06T14:04:02.574869+0000","last_clean":"2024-04-06T14:04:02.574869+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:04:02.574869+0000","last_undegraded":"2024-04-06T14:04:02.574869+0000","last_fullsized":"2024-04-06T14:04:02.574869+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575055+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:04:02.575055+0000","last_peered":"2024-04-06T14:04:02.575055+0000","last_clean":"2024-04-06T14:04:02.575055+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:04:02.575055+0000","last_undegraded":"2024-04-06T14:04:02.575055+0000","last_fullsized":"2024-04-06T14:04:02.575055+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489833+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:04:01.489833+0000","last_peered":"2024-04-06T14:04:01.489833+0000","last_clean":"2024-04-06T14:04:01.489833+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:04:01.489833+0000","last_undegraded":"2024-04-06T14:04:01.489833+0000","last_fullsized":"2024-04-06T14:04:01.489833+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.672268+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:04:01.672268+0000","last_peered":"2024-04-06T14:04:01.672268+0000","last_clean":"2024-04-06T14:04:01.672268+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:04:01.672268+0000","last_undegraded":"2024-04-06T14:04:01.672268+0000","last_fullsized":"2024-04-06T14:04:01.672268+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575443+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:04:02.575443+0000","last_peered":"2024-04-06T14:04:02.575443+0000","last_clean":"2024-04-06T14:04:02.575443+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:04:02.575443+0000","last_undegraded":"2024-04-06T14:04:02.575443+0000","last_fullsized":"2024-04-06T14:04:02.575443+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574980+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:04:02.574980+0000","last_peered":"2024-04-06T14:04:02.574980+0000","last_clean":"2024-04-06T14:04:02.574980+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:04:02.574980+0000","last_undegraded":"2024-04-06T14:04:02.574980+0000","last_fullsized":"2024-04-06T14:04:02.574980+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.574778+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:04:02.574778+0000","last_peered":"2024-04-06T14:04:02.574778+0000","last_clean":"2024-04-06T14:04:02.574778+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:04:02.574778+0000","last_undegraded":"2024-04-06T14:04:02.574778+0000","last_fullsized":"2024-04-06T14:04:02.574778+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575547+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:04:02.575547+0000","last_peered":"2024-04-06T14:04:02.575547+0000","last_clean":"2024-04-06T14:04:02.575547+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:04:02.575547+0000","last_undegraded":"2024-04-06T14:04:02.575547+0000","last_fullsized":"2024-04-06T14:04:02.575547+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":11,"reported_epoch":81,"state":"active+clean","last_fresh":"2024-04-06T14:04:02.575324+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:04:02.575324+0000","last_peered":"2024-04-06T14:04:02.575324+0000","last_clean":"2024-04-06T14:04:02.575324+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:04:02.575324+0000","last_undegraded":"2024-04-06T14:04:02.575324+0000","last_fullsized":"2024-04-06T14:04:02.575324+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":11,"reported_epoch":80,"state":"active+clean","last_fresh":"2024-04-06T14:04:01.489822+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:04:01.489822+0000","last_peered":"2024-04-06T14:04:01.489822+0000","last_clean":"2024-04-06T14:04:01.489822+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:04:01.489822+0000","last_undegraded":"2024-04-06T14:04:01.489822+0000","last_fullsized":"2024-04-06T14:04:01.489822+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2,"ondisk_log_size":2,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":376832,"data_stored":4132576,"data_compressed":246766,"data_compressed_allocated":315392,"data_compressed_original":4096000,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":64907,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":254,"num_read_kb":195,"num_write":150,"num_write_kb":106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":405504,"data_stored":194721,"data_compressed":5499,"data_compressed_allocated":12288,"data_compressed_original":24576,"omap_allocated":0,"internal_metadata":0},"log_size":132,"ondisk_log_size":132,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514516,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31960,"kb_used_data":848,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713192,"statfs":{"total":95995035648,"available":95962308608,"internally_reserved":0,"allocated":868352,"data_stored":417921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.68300000000000005}]},{"osd":1,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.63800000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.76700000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.81100000000000005}]},{"osd":6,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.745},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.73099999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.745,"15min":0.745},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.83499999999999996}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.86499999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.88800000000000001}]},{"osd":10,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.84699999999999998}]}]},{"osd":10,"up_from":68,"seq":292057776150,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28004,"kb_used_data":988,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717148,"statfs":{"total":95995035648,"available":95966359552,"internally_reserved":0,"allocated":1011712,"data_stored":1070416,"data_compressed":37553,"data_compressed_allocated":49152,"data_compressed_original":598016,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.623}]},{"osd":2,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.59599999999999997}]},{"osd":3,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.497}]},{"osd":5,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.64900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.68999999999999995}]},{"osd":9,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.66000000000000003}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.71599999999999997}]}]},{"osd":9,"up_from":63,"seq":270582939674,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27880,"kb_used_data":864,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717272,"statfs":{"total":95995035648,"available":95966486528,"internally_reserved":0,"allocated":884736,"data_stored":418157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,5,6,7,8,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.496}]},{"osd":6,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.38},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.71199999999999997}]},{"osd":8,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.58899999999999997}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.72099999999999997}]}]},{"osd":8,"up_from":56,"seq":240518168605,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27876,"kb_used_data":860,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717276,"statfs":{"total":95995035648,"available":95966490624,"internally_reserved":0,"allocated":880640,"data_stored":417746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54900000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.755,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76300000000000001,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.64700000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77700000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79900000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.623}]},{"osd":5,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74399999999999999,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.37},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.52700000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80900000000000005,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.5,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71399999999999997,"15min":0.71599999999999997},"min":{"1min":0.504,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85899999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.56200000000000006,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.88,"5min":1.028,"15min":1.028},"last":0.64800000000000002}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.61299999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:09 2024","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.58499999999999996}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":7,"up_from":49,"seq":210453397537,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32020,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713132,"statfs":{"total":95995035648,"available":95962247168,"internally_reserved":0,"allocated":929792,"data_stored":1009582,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.377,"15min":0.36899999999999999},"min":{"1min":0.28499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.371},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.312,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55600000000000005,"15min":0.55200000000000005},"min":{"1min":0.46600000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.56599999999999995,"15min":0.55600000000000005},"min":{"1min":0.38500000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.80800000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.496}]},{"osd":3,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.52600000000000002,"15min":0.52000000000000002},"min":{"1min":0.36399999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54800000000000004,"15min":0.54300000000000004},"min":{"1min":0.44800000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.72499999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.73099999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.5,"15min":0.495},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.68100000000000005,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.57799999999999996},"min":{"1min":0.46400000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.61399999999999999,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.442}]},{"osd":6,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.624,"15min":0.61799999999999999},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.57499999999999996,"15min":0.57199999999999995},"min":{"1min":0.42099999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78700000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.61699999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.66300000000000003,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81899999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.58799999999999997}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.68200000000000005}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.625}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]}]},{"osd":6,"up_from":45,"seq":193273528356,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31964,"kb_used_data":852,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713188,"statfs":{"total":95995035648,"available":95962304512,"internally_reserved":0,"allocated":872448,"data_stored":417546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.40899999999999997,"15min":0.41199999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.57399999999999995,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.39700000000000002,"15min":0.39600000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.51000000000000001,"15min":0.502},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.311},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.58899999999999997,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68899999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.44},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.46899999999999997,"15min":0.47099999999999997},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.433},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.621}]},{"osd":5,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.83599999999999997,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.63600000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54900000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.65700000000000003,"15min":0.65500000000000003},"min":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.48399999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.495,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.79900000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.66100000000000003,"15min":0.65400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.65900000000000003}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51900000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.56100000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574902,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27924,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717228,"statfs":{"total":95995035648,"available":95966441472,"internally_reserved":0,"allocated":929792,"data_stored":478339,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.35899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.48799999999999999,"15min":0.47299999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Apr 6 14:03:55 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.51800000000000002,"15min":0.496},"min":{"1min":0.375,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75600000000000001,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.52400000000000002,"15min":0.50700000000000001},"min":{"1min":0.39900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.70399999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.50700000000000001,"15min":0.48699999999999999},"min":{"1min":0.35899999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.52400000000000002,"15min":0.51000000000000001},"min":{"1min":0.40300000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.76000000000000001,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46100000000000002,"15min":0.44700000000000001},"min":{"1min":0.38500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.41799999999999998},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.57899999999999996,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.79200000000000004,"5min":0.877,"15min":0.877},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56899999999999995,"15min":0.55700000000000005},"min":{"1min":0.48499999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.79200000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58899999999999997,"15min":0.58499999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.624},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.628,"15min":0.621},"min":{"1min":0.496,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:04:20 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.56100000000000005,"15min":0.54900000000000004},"min":{"1min":0.48399999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80600000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.626,"15min":0.61599999999999999},"min":{"1min":0.38100000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.875}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.70099999999999996,"15min":0.69499999999999995},"min":{"1min":0.53400000000000003,"5min":0.503,"15min":0.503},"max":{"1min":0.90700000000000003,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.65700000000000003,"15min":0.65000000000000002},"min":{"1min":0.56499999999999995,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.83799999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.82799999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.84299999999999997}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.84999999999999998}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.89800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27888,"kb_used_data":872,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717264,"statfs":{"total":95995035648,"available":95966478336,"internally_reserved":0,"allocated":892928,"data_stored":418817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50700000000000001,"15min":0.51700000000000002},"min":{"1min":0.32700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68500000000000005,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.32800000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56599999999999995,"15min":0.56499999999999995},"min":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79600000000000004,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:03:14 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.26500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.67200000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.308,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63800000000000001,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.50900000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.497,"15min":0.499},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65100000000000002,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60899999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54000000000000004,"15min":0.54200000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.71099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54700000000000004,"15min":0.54300000000000004},"min":{"1min":0.443,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46700000000000003,"15min":0.46899999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70799999999999996,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.35799999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.502,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.65500000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.70199999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:03:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.504},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.627}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.0669999999999999,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.67300000000000004}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.64900000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":932,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":954368,"data_stored":479641,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.47299999999999998,"15min":0.45300000000000001},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.436},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45600000000000002,"15min":0.439},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52200000000000002,"15min":0.51400000000000001},"min":{"1min":0.39500000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.53500000000000003,"15min":0.51700000000000002},"min":{"1min":0.438,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.69199999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.53500000000000003,"15min":0.53200000000000003},"min":{"1min":0.39100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.74199999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Apr 6 14:03:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.51900000000000002,"15min":0.51100000000000001},"min":{"1min":0.432,"5min":0.248,"15min":0.248},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.55200000000000005,"15min":0.54000000000000004},"min":{"1min":0.46999999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.76500000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:03:46 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.438,"15min":0.42999999999999999},"min":{"1min":0.32700000000000001,"5min":0.216,"15min":0.216},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.42399999999999999,"15min":0.41499999999999998},"min":{"1min":0.31900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.66800000000000004}]},{"osd":6,"last update":"Sat Apr 6 14:04:04 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.60599999999999998,"15min":0.59999999999999998},"min":{"1min":0.44,"5min":0.372,"15min":0.372},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.44800000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.747},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61799999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:03:54 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.59899999999999998}]},{"osd":10,"last update":"Sat Apr 6 14:04:11 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.81499999999999995}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77700000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149743,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36076,"kb_used_data":868,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93709076,"statfs":{"total":95995035648,"available":95958093824,"internally_reserved":0,"allocated":888832,"data_stored":417857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34100000000000003,"15min":0.33500000000000002},"min":{"1min":0.26300000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.34300000000000003,"15min":0.33300000000000002},"min":{"1min":0.27000000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.48499999999999999,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.504,"15min":0.48499999999999999},"min":{"1min":0.441,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83399999999999996,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.505,"15min":0.48899999999999999},"min":{"1min":0.41499999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.71099999999999997,"5min":0.872,"15min":0.872},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53500000000000003,"15min":0.52500000000000002},"min":{"1min":0.41199999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.67100000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54600000000000004,"15min":0.54100000000000004},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.79100000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65200000000000002,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Apr 6 14:03:45 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.46700000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Apr 6 14:04:07 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.443,"5min":0.49099999999999999,"15min":0.499},"min":{"1min":0.32300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.60999999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:04:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56399999999999995,"15min":0.56299999999999994},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.754,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.46999999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.66600000000000004}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:03:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.42299999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:13 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.71399999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68600000000000005}]}]},{"osd":4,"up_from":33,"seq":141733920812,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27872,"kb_used_data":856,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717280,"statfs":{"total":95995035648,"available":95966494720,"internally_reserved":0,"allocated":876544,"data_stored":418328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35399999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.40200000000000002,"15min":0.378},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.70199999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":1.036,"15min":1.036},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.55800000000000005,"15min":0.53600000000000003},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.755}]},{"osd":3,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.50700000000000001},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76500000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.47999999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68500000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Apr 6 14:03:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.55600000000000005,"15min":0.54900000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.59399999999999997,"15min":0.58399999999999996},"min":{"1min":0.42899999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.71399999999999997}]},{"osd":6,"last update":"Sat Apr 6 14:04:01 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.55400000000000005,"15min":0.53800000000000003},"min":{"1min":0.48299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.81799999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.52400000000000002,"15min":0.50800000000000001},"min":{"1min":0.45000000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.44900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57699999999999996,"15min":0.56299999999999994},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.78000000000000003}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.68000000000000005,"15min":0.67700000000000005},"min":{"1min":0.52200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85799999999999998,"5min":0.879,"15min":0.879},"last":0.745},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73899999999999999,"15min":0.746},"min":{"1min":0.57999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81999999999999995,"5min":1.694,"15min":1.694},"last":0.80200000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:03:53 2024","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.79100000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:04:10 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.77000000000000002}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83499999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32036,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713116,"statfs":{"total":95995035648,"available":95962230784,"internally_reserved":0,"allocated":946176,"data_stored":1009252,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.49299999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.48999999999999999,"15min":0.47699999999999998},"min":{"1min":0.36599999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.51100000000000001,"15min":0.497},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.441},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.499,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.38300000000000001,"15min":0.37},"min":{"1min":0.29999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.317,"5min":0.223,"15min":0.223},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52200000000000002,"15min":0.51700000000000002},"min":{"1min":0.41099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.52900000000000003,"15min":0.51600000000000001},"min":{"1min":0.48699999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.58799999999999997}]},{"osd":4,"last update":"Sat Apr 6 14:03:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.57199999999999995,"15min":0.56200000000000006},"min":{"1min":0.45400000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.56899999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.79700000000000004,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:04:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.59499999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.83199999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.627,"15min":0.61799999999999999},"min":{"1min":0.48299999999999998,"5min":0.378,"15min":0.378},"max":{"1min":0.876,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Apr 6 14:04:19 2024","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.63,"15min":0.62},"min":{"1min":0.503,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.63100000000000001,"15min":0.62},"min":{"1min":0.52100000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.89700000000000002,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.52700000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.625,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.68400000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.88200000000000001,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.64000000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:03:52 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0529999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.61499999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:04:08 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.626}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":34083,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":61000,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":90112,"data_stored":62302,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":62923,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:04:49.096 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-06T14:04:49.096 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-06T14:04:49.097 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-06T14:04:49.097 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-06T14:04:49.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:49 smithi090 ceph-mon[43420]: pgmap v243: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:49.322 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:49.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:49 smithi105 ceph-mon[46969]: pgmap v243: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:49.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:49 smithi134 ceph-mon[47549]: pgmap v243: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s rd, 170 B/s wr, 5 op/s 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 -- 172.21.15.90:0/2978593049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e00105d10 msgr2=0x7f0e001060f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2978593049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e00105d10 0x7f0e001060f0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f0de8009990 tx=0x7f0de8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 -- 172.21.15.90:0/2978593049 shutdown_connections 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2978593049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e001006a0 0x7f0e0010cb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2978593049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e00105d10 0x7f0e001060f0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2978593049 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 0x7f0e00100160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 -- 172.21.15.90:0/2978593049 >> 172.21.15.90:0/2978593049 conn(0x7f0e00075d90 msgr2=0x7f0e000761a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:50.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 -- 172.21.15.90:0/2978593049 shutdown_connections 2024-04-06T14:04:50.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.376+0000 7f0e0524f700 1 -- 172.21.15.90:0/2978593049 wait complete. 2024-04-06T14:04:50.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 Processor -- start 2024-04-06T14:04:50.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 -- start start 2024-04-06T14:04:50.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 0x7f0e001971c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:50.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:50.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e00105d10 0x7f0e0019bab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e0019c0c0 con 0x7f0e001006a0 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e0019c230 con 0x7f0e00105d10 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.377+0000 7f0e0524f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e0019c3f0 con 0x7f0e000ffce0 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51948/0 (socket says 172.21.15.90:51948) 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 -- 172.21.15.90:0/2027008660 learned_addr learned my addr 172.21.15.90:0/2027008660 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dff59e700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e00105d10 0x7f0e0019bab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfed9d700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 0x7f0e001971c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 -- 172.21.15.90:0/2027008660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 msgr2=0x7f0e001971c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 0x7f0e001971c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 -- 172.21.15.90:0/2027008660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e00105d10 msgr2=0x7f0e0019bab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e00105d10 0x7f0e0019bab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 -- 172.21.15.90:0/2027008660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0de80095f0 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0dfe59c700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f0df000c920 tx=0x7f0df000cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0df0007b00 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0e0019c6d0 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0df000ce70 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0df0005460 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.378+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0e001a3f30 con 0x7f0e001006a0 2024-04-06T14:04:50.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.379+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0e001013c0 con 0x7f0e001006a0 2024-04-06T14:04:50.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.380+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f0df00055c0 con 0x7f0e001006a0 2024-04-06T14:04:50.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.381+0000 7f0df7fff700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 0x7f0dec0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:50.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.381+0000 7f0dfed9d700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 0x7f0dec0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:50.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.381+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f0df00cc8c0 con 0x7f0e001006a0 2024-04-06T14:04:50.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.381+0000 7f0dfed9d700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 0x7f0dec0a3670 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f0de8005850 tx=0x7f0de801a040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:50.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.385+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0df00960a0 con 0x7f0e001006a0 2024-04-06T14:04:50.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:50 smithi105 ceph-mon[46969]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:50.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:50 smithi090 ceph-mon[43420]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.619+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f0e0004ea80 con 0x7f0e001006a0 2024-04-06T14:04:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.619+0000 7f0df7fff700 1 -- 172.21.15.90:0/2027008660 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f0df001d080 con 0x7f0e001006a0 2024-04-06T14:04:50.620 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:50.620 INFO:teuthology.orchestra.run.smithi090.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 msgr2=0x7f0dec0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 0x7f0dec0a3670 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f0de8005850 tx=0x7f0de801a040 comp rx=0 tx=0).stop 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 msgr2=0x7f0e00197700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f0df000c920 tx=0x7f0df000cc30 comp rx=0 tx=0).stop 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 shutdown_connections 2024-04-06T14:04:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0dec0a11b0 0x7f0dec0a3670 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0e00105d10 0x7f0e0019bab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0e000ffce0 0x7f0e001971c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 --2- 172.21.15.90:0/2027008660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0e001006a0 0x7f0e00197700 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.621+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 >> 172.21.15.90:0/2027008660 conn(0x7f0e00075d90 msgr2=0x7f0e000689a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.622+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 shutdown_connections 2024-04-06T14:04:50.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:50.622+0000 7f0e0524f700 1 -- 172.21.15.90:0/2027008660 wait complete. 2024-04-06T14:04:50.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:50 smithi134 ceph-mon[47549]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:04:51.126 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-06T14:04:51.126 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:51.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 -- 172.21.15.90:0/3297022510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 msgr2=0x7f59541011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 --2- 172.21.15.90:0/3297022510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f59541011c0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f59480099c0 tx=0x7f5948009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 -- 172.21.15.90:0/3297022510 shutdown_connections 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 --2- 172.21.15.90:0/3297022510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5954101700 0x7f5954107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 --2- 172.21.15.90:0/3297022510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f59540ffb70 0x7f59540fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 --2- 172.21.15.90:0/3297022510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f59541011c0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 -- 172.21.15.90:0/3297022510 >> 172.21.15.90:0/3297022510 conn(0x7f59540fb100 msgr2=0x7f59540fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 -- 172.21.15.90:0/3297022510 shutdown_connections 2024-04-06T14:04:51.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.254+0000 7f5958a05700 1 -- 172.21.15.90:0/3297022510 wait complete. 2024-04-06T14:04:51.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 Processor -- start 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 -- start start 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f59540ffb70 0x7f5954068940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5954101700 0x7f595419df00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f595419e6f0 con 0x7f5954100d60 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f595419e860 con 0x7f5954101700 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5958a05700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f595419ea20 con 0x7f59540ffb70 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51962/0 (socket says 172.21.15.90:51962) 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 -- 172.21.15.90:0/3572532127 learned_addr learned my addr 172.21.15.90:0/3572532127 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f595259c700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f59540ffb70 0x7f5954068940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5952d9d700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5954101700 0x7f595419df00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 -- 172.21.15.90:0/3572532127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f59540ffb70 msgr2=0x7f5954068940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f59540ffb70 0x7f5954068940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 -- 172.21.15.90:0/3572532127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5954101700 msgr2=0x7f595419df00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5954101700 0x7f595419df00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 -- 172.21.15.90:0/3572532127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59480095f0 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.255+0000 7f5951d9b700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f59480036f0 tx=0x7f59480047b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.256+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f594801d070 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.256+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5948003f90 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.256+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5948022c20 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.256+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f595419ec40 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.256+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f595419f0a0 con 0x7f5954100d60 2024-04-06T14:04:51.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.257+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5954106f40 con 0x7f5954100d60 2024-04-06T14:04:51.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.258+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f5948022470 con 0x7f5954100d60 2024-04-06T14:04:51.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.258+0000 7f59437fe700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 0x7f593c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.258+0000 7f595259c700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 0x7f593c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.258+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f59480ce140 con 0x7f5954100d60 2024-04-06T14:04:51.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.259+0000 7f595259c700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 0x7f593c0a3670 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f5954100bc0 tx=0x7f594400b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.262+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f59480979b0 con 0x7f5954100d60 2024-04-06T14:04:51.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.463+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f595410b380 con 0x7f5954100d60 2024-04-06T14:04:51.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.464+0000 7f59437fe700 1 -- 172.21.15.90:0/3572532127 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7f594809afd0 con 0x7f5954100d60 2024-04-06T14:04:51.465 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:51.465 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.466+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 msgr2=0x7f593c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.466+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 0x7f593c0a3670 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f5954100bc0 tx=0x7f594400b410 comp rx=0 tx=0).stop 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.466+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 msgr2=0x7f595419b8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.466+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f59480036f0 tx=0x7f59480047b0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 shutdown_connections 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f593c0a11b0 0x7f593c0a3670 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5954101700 0x7f595419df00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f59540ffb70 0x7f5954068940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 --2- 172.21.15.90:0/3572532127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5954100d60 0x7f595419b8e0 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 >> 172.21.15.90:0/3572532127 conn(0x7f59540fb100 msgr2=0x7f5954104080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:51.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 shutdown_connections 2024-04-06T14:04:51.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.467+0000 7f5958a05700 1 -- 172.21.15.90:0/3572532127 wait complete. 2024-04-06T14:04:51.478 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:51 smithi090 ceph-mon[43420]: pgmap v244: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:51.478 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:51 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2027008660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-06T14:04:51.480 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:04:51.480 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:04:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:51 smithi105 ceph-mon[46969]: pgmap v244: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:51 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2027008660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.605+0000 7f793032a700 1 -- 172.21.15.90:0/2571032809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7928106a50 msgr2=0x7f7928108ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.605+0000 7f793032a700 1 --2- 172.21.15.90:0/2571032809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7928106a50 0x7f7928108ec0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f792000b0a0 tx=0x7f792000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 -- 172.21.15.90:0/2571032809 shutdown_connections 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 --2- 172.21.15.90:0/2571032809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7928106a50 0x7f7928108ec0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 --2- 172.21.15.90:0/2571032809 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f79281017f0 0x7f7928103be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 --2- 172.21.15.90:0/2571032809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928106510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 -- 172.21.15.90:0/2571032809 >> 172.21.15.90:0/2571032809 conn(0x7f79280fb200 msgr2=0x7f79280fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 -- 172.21.15.90:0/2571032809 shutdown_connections 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 -- 172.21.15.90:0/2571032809 wait complete. 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 Processor -- start 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.606+0000 7f793032a700 1 -- start start 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928197320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7928106a50 0x7f792819e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f792819e8c0 con 0x7f79281017f0 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f792819ea00 con 0x7f7928106a50 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f792819eb90 con 0x7f7928104120 2024-04-06T14:04:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51970/0 (socket says 172.21.15.90:51970) 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 -- 172.21.15.90:0/467311052 learned_addr learned my addr 172.21.15.90:0/467311052 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792d8c5700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928197320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e8c7700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7928106a50 0x7f792819e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 -- 172.21.15.90:0/467311052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 msgr2=0x7f7928197320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928197320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 -- 172.21.15.90:0/467311052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7928106a50 msgr2=0x7f792819e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7928106a50 0x7f792819e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 -- 172.21.15.90:0/467311052 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7920009d00 con 0x7f79281017f0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792d8c5700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928197320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f792e0c6700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f792400e9a0 tx=0x7f792400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7924009790 con 0x7f79281017f0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f792400cbd0 con 0x7f79281017f0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f792819ee70 con 0x7f79281017f0 2024-04-06T14:04:51.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7924018680 con 0x7f79281017f0 2024-04-06T14:04:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.607+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f792819f290 con 0x7f79281017f0 2024-04-06T14:04:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.608+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79280fcd80 con 0x7f79281017f0 2024-04-06T14:04:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.610+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f792400cd40 con 0x7f79281017f0 2024-04-06T14:04:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.611+0000 7f791f7fe700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 0x7f79140a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.611+0000 7f792d8c5700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 0x7f79140a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.611+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f7924014070 con 0x7f79281017f0 2024-04-06T14:04:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.611+0000 7f792d8c5700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 0x7f79140a3670 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f7918005950 tx=0x7f79180058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.614+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f79240965f0 con 0x7f79281017f0 2024-04-06T14:04:51.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:51 smithi134 ceph-mon[47549]: pgmap v244: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.5 KiB/s rd, 170 B/s wr, 6 op/s 2024-04-06T14:04:51.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:51 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2027008660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-06T14:04:51.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.812+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f792810c750 con 0x7f79281017f0 2024-04-06T14:04:51.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.812+0000 7f791f7fe700 1 -- 172.21.15.90:0/467311052 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) v1 ==== 93+0+10 (secure 0 0 0) 0x7f7924099c10 con 0x7f79281017f0 2024-04-06T14:04:51.813 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:04:51.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 msgr2=0x7f79140a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 0x7f79140a3670 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f7918005950 tx=0x7f79180058e0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 msgr2=0x7f7928196de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f792400e9a0 tx=0x7f792400ecb0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 shutdown_connections 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f79140a11b0 0x7f79140a3670 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7928106a50 0x7f792819e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f79281017f0 0x7f7928196de0 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 --2- 172.21.15.90:0/467311052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7928104120 0x7f7928197320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 >> 172.21.15.90:0/467311052 conn(0x7f79280fb200 msgr2=0x7f7928102230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 shutdown_connections 2024-04-06T14:04:51.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.814+0000 7f793032a700 1 -- 172.21.15.90:0/467311052 wait complete. 2024-04-06T14:04:51.827 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:04:51.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2429171609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 msgr2=0x7fd5e81080b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2429171609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e81080b0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fd5e000b0d0 tx=0x7fd5e000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2429171609 shutdown_connections 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2429171609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e81080b0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2429171609 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 0x7fd5e8100090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2429171609 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 0x7fd5e81012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2429171609 >> 172.21.15.90:0/2429171609 conn(0x7fd5e80fb200 msgr2=0x7fd5e80fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2429171609 shutdown_connections 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.951+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2429171609 wait complete. 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 Processor -- start 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 -- start start 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 0x7fd5e8196e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 0x7fd5e8197380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5e819e970 con 0x7fd5e8101800 2024-04-06T14:04:51.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5e819eab0 con 0x7fd5e8100e60 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5efb9c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5e819ec40 con 0x7fd5e80ffc70 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51978/0 (socket says 172.21.15.90:51978) 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 -- 172.21.15.90:0/2139398426 learned_addr learned my addr 172.21.15.90:0/2139398426 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ed137700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 0x7fd5e8197380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ed938700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 0x7fd5e8196e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 -- 172.21.15.90:0/2139398426 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 msgr2=0x7fd5e8196e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 0x7fd5e8196e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 -- 172.21.15.90:0/2139398426 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 msgr2=0x7fd5e8197380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 0x7fd5e8197380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.952+0000 7fd5ee139700 1 -- 172.21.15.90:0/2139398426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5e0009d00 con 0x7fd5e8101800 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5ee139700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fd5e0007bd0 tx=0x7fd5e0009440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5e000e040 con 0x7fd5e8101800 2024-04-06T14:04:51.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd5e0012dc0 con 0x7fd5e8101800 2024-04-06T14:04:51.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5e819eec0 con 0x7fd5e8101800 2024-04-06T14:04:51.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5e001b450 con 0x7fd5e8101800 2024-04-06T14:04:51.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.953+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5e819f2e0 con 0x7fd5e8101800 2024-04-06T14:04:51.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.954+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd5e8067b00 con 0x7fd5e8101800 2024-04-06T14:04:51.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.955+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fd5e0012900 con 0x7fd5e8101800 2024-04-06T14:04:51.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.955+0000 7fd5daffd700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 0x7fd5d40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:51.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.955+0000 7fd5ed938700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 0x7fd5d40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:51.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.955+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fd5e00cee10 con 0x7fd5e8101800 2024-04-06T14:04:51.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.956+0000 7fd5ed938700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 0x7fd5d40a3670 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fd5e8100cc0 tx=0x7fd5dc005cf0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:51.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:51.959+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd5e0098790 con 0x7fd5e8101800 2024-04-06T14:04:52.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.158+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fd5e810b700 con 0x7fd5e8101800 2024-04-06T14:04:52.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.159+0000 7fd5daffd700 1 -- 172.21.15.90:0/2139398426 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v81) v1 ==== 104+0+11 (secure 0 0 0) 0x7fd5e009bdb0 con 0x7fd5e8101800 2024-04-06T14:04:52.159 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:04:52.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 msgr2=0x7fd5d40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 0x7fd5d40a3670 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fd5e8100cc0 tx=0x7fd5dc005cf0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 msgr2=0x7fd5e819e3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fd5e0007bd0 tx=0x7fd5e0009440 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 shutdown_connections 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd5d40a11b0 0x7fd5d40a3670 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd5e8101800 0x7fd5e819e3a0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd5e80ffc70 0x7fd5e8196e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 --2- 172.21.15.90:0/2139398426 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5e8100e60 0x7fd5e8197380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 >> 172.21.15.90:0/2139398426 conn(0x7fd5e80fb200 msgr2=0x7fd5e80fccb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 shutdown_connections 2024-04-06T14:04:52.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.161+0000 7fd5efb9c700 1 -- 172.21.15.90:0/2139398426 wait complete. 2024-04-06T14:04:52.174 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/323492639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54101f80 msgr2=0x7f6e541023e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/323492639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54101f80 0x7f6e541023e0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f6e500099c0 tx=0x7f6e50009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/323492639 shutdown_connections 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/323492639 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54102920 0x7f6e541091d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/323492639 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54100d90 0x7f6e541011b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/323492639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54101f80 0x7f6e541023e0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/323492639 >> 172.21.15.90:0/323492639 conn(0x7f6e540fc300 msgr2=0x7f6e540fe740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:52.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/323492639 shutdown_connections 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.300+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/323492639 wait complete. 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 Processor -- start 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 -- start start 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 0x7f6e54196d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54101f80 0x7f6e541972d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e5419e880 con 0x7f6e54102920 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e5419e9f0 con 0x7f6e54101f80 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e5419ebb0 con 0x7f6e54100d90 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:51996/0 (socket says 172.21.15.90:51996) 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 -- 172.21.15.90:0/2498089022 learned_addr learned my addr 172.21.15.90:0/2498089022 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e58855700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54101f80 0x7f6e541972d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59056700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 0x7f6e54196d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 -- 172.21.15.90:0/2498089022 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 msgr2=0x7f6e54196d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 0x7f6e54196d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 -- 172.21.15.90:0/2498089022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54101f80 msgr2=0x7f6e541972d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54101f80 0x7f6e541972d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 -- 172.21.15.90:0/2498089022 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e500095f0 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59056700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 0x7f6e54196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e59857700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f6e4c00b970 tx=0x7f6e4c00bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e4c00c930 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6e4c014440 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.301+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e5419ee90 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.302+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e4c0075d0 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.302+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e5419f2b0 con 0x7f6e54102920 2024-04-06T14:04:52.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.302+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6e54108230 con 0x7f6e54102920 2024-04-06T14:04:52.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.304+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f6e4c0145b0 con 0x7f6e54102920 2024-04-06T14:04:52.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.305+0000 7f6e4a7fc700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 0x7f6e400a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.305+0000 7f6e59056700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 0x7f6e400a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.305+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f6e4c006e40 con 0x7f6e54102920 2024-04-06T14:04:52.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.305+0000 7f6e59056700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 0x7f6e400a36f0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f6e44005fd0 tx=0x7f6e44005f20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:52.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.308+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6e4c095b30 con 0x7f6e54102920 2024-04-06T14:04:52.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3572532127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:52.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/467311052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:52 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2139398426' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.508+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f6e5410c730 con 0x7f6e54102920 2024-04-06T14:04:52.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.509+0000 7f6e4a7fc700 1 -- 172.21.15.90:0/2498089022 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v81) v1 ==== 100+0+11 (secure 0 0 0) 0x7f6e4c0230a0 con 0x7f6e54102920 2024-04-06T14:04:52.509 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 msgr2=0x7f6e400a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 0x7f6e400a36f0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f6e44005fd0 tx=0x7f6e44005f20 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 msgr2=0x7f6e5419e2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f6e4c00b970 tx=0x7f6e4c00bc80 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 shutdown_connections 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f6e400a1230 0x7f6e400a36f0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6e54102920 0x7f6e5419e2b0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6e54100d90 0x7f6e54196d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 --2- 172.21.15.90:0/2498089022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6e54101f80 0x7f6e541972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 >> 172.21.15.90:0/2498089022 conn(0x7f6e540fc300 msgr2=0x7f6e54105370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 shutdown_connections 2024-04-06T14:04:52.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.511+0000 7f6e5b2ba700 1 -- 172.21.15.90:0/2498089022 wait complete. 2024-04-06T14:04:52.523 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3572532127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:52.523 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/467311052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.523 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:52 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2139398426' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.524 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:04:52.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3572532127' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:52.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/467311052' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:52 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2139398426' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:52.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 -- 172.21.15.90:0/1700394353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0100790 msgr2=0x7f76c0108030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 --2- 172.21.15.90:0/1700394353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0100790 0x7f76c0108030 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f76b800b0a0 tx=0x7f76b800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 -- 172.21.15.90:0/1700394353 shutdown_connections 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 --2- 172.21.15.90:0/1700394353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0100790 0x7f76c0108030 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 --2- 172.21.15.90:0/1700394353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0101180 0x7f76c01015a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 --2- 172.21.15.90:0/1700394353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 0x7f76c0100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 -- 172.21.15.90:0/1700394353 >> 172.21.15.90:0/1700394353 conn(0x7f76c00fa880 msgr2=0x7f76c00fcca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 -- 172.21.15.90:0/1700394353 shutdown_connections 2024-04-06T14:04:52.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.645+0000 7f76c73a9700 1 -- 172.21.15.90:0/1700394353 wait complete. 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 Processor -- start 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 -- start start 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 0x7f76c0195ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0100790 0x7f76c01960e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76c019d6d0 con 0x7f76c0101180 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76c019d810 con 0x7f76c00ffdd0 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c73a9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f76c019d9a0 con 0x7f76c0100790 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5946700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5145700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 0x7f76c0195ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5946700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52016/0 (socket says 172.21.15.90:52016) 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5946700 1 -- 172.21.15.90:0/3057046340 learned_addr learned my addr 172.21.15.90:0/3057046340 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:52.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c4944700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0100790 0x7f76c01960e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5946700 1 -- 172.21.15.90:0/3057046340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0100790 msgr2=0x7f76c01960e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.646+0000 7f76c5946700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0100790 0x7f76c01960e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c5946700 1 -- 172.21.15.90:0/3057046340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 msgr2=0x7f76c0195ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c5946700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 0x7f76c0195ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c5946700 1 -- 172.21.15.90:0/3057046340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f76b8009d00 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c5946700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f76b8009030 tx=0x7f76b800bab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b800e040 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f76b8004c50 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f76c019dc20 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f76b80052a0 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.647+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f76c019e0f0 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.648+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f76c0106fa0 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.649+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f76b8005400 con 0x7f76c0101180 2024-04-06T14:04:52.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.649+0000 7f76b67fc700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 0x7f76ac0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:52.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.649+0000 7f76c5145700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 0x7f76ac0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:52.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.650+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f76b80ce620 con 0x7f76c0101180 2024-04-06T14:04:52.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.650+0000 7f76c5145700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 0x7f76ac0a36f0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f76c0067be0 tx=0x7f76b0009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:52.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.653+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f76b8097e10 con 0x7f76c0101180 2024-04-06T14:04:52.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.894+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f76c0067190 con 0x7f76c0101180 2024-04-06T14:04:52.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.894+0000 7f76b67fc700 1 -- 172.21.15.90:0/3057046340 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f76b809b430 con 0x7f76c0101180 2024-04-06T14:04:52.895 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:52.896 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 msgr2=0x7f76ac0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 0x7f76ac0a36f0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f76c0067be0 tx=0x7f76b0009260 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 msgr2=0x7f76c019d100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f76b8009030 tx=0x7f76b800bab0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 shutdown_connections 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f76ac0a1230 0x7f76ac0a36f0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f76c0101180 0x7f76c019d100 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f76c00ffdd0 0x7f76c0195ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 --2- 172.21.15.90:0/3057046340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f76c0100790 0x7f76c01960e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:52.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.896+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 >> 172.21.15.90:0/3057046340 conn(0x7f76c00fa880 msgr2=0x7f76c01040e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:52.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.897+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 shutdown_connections 2024-04-06T14:04:52.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:52.897+0000 7f76c73a9700 1 -- 172.21.15.90:0/3057046340 wait complete. 2024-04-06T14:04:52.901 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:04:52.910 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 -- 172.21.15.90:0/110023085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614102810 msgr2=0x7f56141090c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 --2- 172.21.15.90:0/110023085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614102810 0x7f56141090c0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f560c00b0d0 tx=0x7f560c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 -- 172.21.15.90:0/110023085 shutdown_connections 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 --2- 172.21.15.90:0/110023085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614102810 0x7f56141090c0 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 --2- 172.21.15.90:0/110023085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614100c80 0x7f56141010a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 --2- 172.21.15.90:0/110023085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614101e70 0x7f56141022d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 -- 172.21.15.90:0/110023085 >> 172.21.15.90:0/110023085 conn(0x7f56140fc210 msgr2=0x7f56140fe630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 -- 172.21.15.90:0/110023085 shutdown_connections 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.032+0000 7f561b69f700 1 -- 172.21.15.90:0/110023085 wait complete. 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 Processor -- start 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 -- start start 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 0x7f5614198f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 0x7f56141a04f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56141a0ac0 con 0x7f5614101e70 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56141a0c00 con 0x7f5614100c80 2024-04-06T14:04:53.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561b69f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56141a0d90 con 0x7f5614102810 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52042/0 (socket says 172.21.15.90:52042) 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 -- 172.21.15.90:0/3251820757 learned_addr learned my addr 172.21.15.90:0/3251820757 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561943b700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 0x7f5614198f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5619c3c700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 0x7f56141a04f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 -- 172.21.15.90:0/3251820757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 msgr2=0x7f56141a04f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 0x7f56141a04f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 -- 172.21.15.90:0/3251820757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 msgr2=0x7f5614198f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 0x7f5614198f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5618c3a700 1 -- 172.21.15.90:0/3251820757 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f560c009d00 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f561943b700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 0x7f5614198f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.033+0000 7f5619c3c700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 0x7f56141a04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f5618c3a700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f560800e9a0 tx=0x7f560800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5608009790 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56141a1070 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f560800cbd0 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5608018680 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.034+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56141a1650 con 0x7f5614101e70 2024-04-06T14:04:53.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.035+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5614067190 con 0x7f5614101e70 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.036+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f560800cd40 con 0x7f5614101e70 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.036+0000 7f56067fc700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 0x7f56000a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.036+0000 7f561943b700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 0x7f56000a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.037+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f5608014070 con 0x7f5614101e70 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.037+0000 7f561943b700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 0x7f56000a3670 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f5614101cd0 tx=0x7f5610008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.040+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5608096890 con 0x7f5614101e70 2024-04-06T14:04:53.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.240+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f561410c530 con 0x7f5614101e70 2024-04-06T14:04:53.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.242+0000 7f56067fc700 1 -- 172.21.15.90:0/3251820757 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7f5608099eb0 con 0x7f5614101e70 2024-04-06T14:04:53.243 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:53.243 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 msgr2=0x7f56000a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 0x7f56000a3670 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f5614101cd0 tx=0x7f5610008040 comp rx=0 tx=0).stop 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 msgr2=0x7f56141994d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f560800e9a0 tx=0x7f560800ed60 comp rx=0 tx=0).stop 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 shutdown_connections 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f56000a11b0 0x7f56000a3670 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5614102810 0x7f56141a04f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5614100c80 0x7f5614198f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 --2- 172.21.15.90:0/3251820757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5614101e70 0x7f56141994d0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 >> 172.21.15.90:0/3251820757 conn(0x7f56140fc210 msgr2=0x7f5614072fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 shutdown_connections 2024-04-06T14:04:53.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.244+0000 7f561b69f700 1 -- 172.21.15.90:0/3251820757 wait complete. 2024-04-06T14:04:53.250 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:53 smithi090 ceph-mon[43420]: pgmap v245: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:53.250 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:53 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2498089022' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:53.250 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:53 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3057046340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.251 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:53 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3251820757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.258 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:04:53.259 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:04:53.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- 172.21.15.90:0/316432842 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 msgr2=0x7f27a81080d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 --2- 172.21.15.90:0/316432842 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a81080d0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f27a000b0d0 tx=0x7f27a000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- 172.21.15.90:0/316432842 shutdown_connections 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 --2- 172.21.15.90:0/316432842 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a81080d0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 --2- 172.21.15.90:0/316432842 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 0x7f27a81000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 --2- 172.21.15.90:0/316432842 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 0x7f27a81012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- 172.21.15.90:0/316432842 >> 172.21.15.90:0/316432842 conn(0x7f27a80fb200 msgr2=0x7f27a80fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- 172.21.15.90:0/316432842 shutdown_connections 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- 172.21.15.90:0/316432842 wait complete. 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 Processor -- start 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.383+0000 7f27aca07700 1 -- start start 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 0x7f27a8196df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 0x7f27a8197330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27a819e8e0 con 0x7f27a8101820 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27a819ea50 con 0x7f27a80ffc90 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27a819ec10 con 0x7f27a8100e80 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38472/0 (socket says 172.21.15.90:38472) 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 -- 172.21.15.90:0/426104169 learned_addr learned my addr 172.21.15.90:0/426104169 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a659c700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 0x7f27a8196df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a5d9b700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 0x7f27a8197330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 -- 172.21.15.90:0/426104169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 msgr2=0x7f27a8197330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 0x7f27a8197330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 -- 172.21.15.90:0/426104169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 msgr2=0x7f27a8196df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 0x7f27a8196df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 -- 172.21.15.90:0/426104169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27a0009d00 con 0x7f27a8101820 2024-04-06T14:04:53.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27a6d9d700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f27a0007bd0 tx=0x7f27a0009440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27a000e040 con 0x7f27a8101820 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f27a001dd50 con 0x7f27a8101820 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f27a819ee90 con 0x7f27a8101820 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27a00123f0 con 0x7f27a8101820 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.384+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27a819f250 con 0x7f27a8101820 2024-04-06T14:04:53.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.385+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f27a8067f50 con 0x7f27a8101820 2024-04-06T14:04:53.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.386+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f27a001d890 con 0x7f27a8101820 2024-04-06T14:04:53.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.387+0000 7f27977fe700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 0x7f27900a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.387+0000 7f27a659c700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 0x7f27900a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.387+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f27a00cee50 con 0x7f27a8101820 2024-04-06T14:04:53.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.387+0000 7f27a659c700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 0x7f27900a3670 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f27a8198270 tx=0x7f279800b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.390+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f27a00986c0 con 0x7f27a8101820 2024-04-06T14:04:53.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.585+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f27a810b4b0 con 0x7f27a8101820 2024-04-06T14:04:53.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.585+0000 7f27977fe700 1 -- 172.21.15.90:0/426104169 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v81) v1 ==== 93+0+10 (secure 0 0 0) 0x7f27a009bce0 con 0x7f27a8101820 2024-04-06T14:04:53.586 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:04:53.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 msgr2=0x7f27900a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 0x7f27900a3670 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f27a8198270 tx=0x7f279800b410 comp rx=0 tx=0).stop 2024-04-06T14:04:53.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 msgr2=0x7f27a819e310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f27a0007bd0 tx=0x7f27a0009440 comp rx=0 tx=0).stop 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 shutdown_connections 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f27900a11b0 0x7f27900a3670 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27a8101820 0x7f27a819e310 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a80ffc90 0x7f27a8196df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 --2- 172.21.15.90:0/426104169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27a8100e80 0x7f27a8197330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.587+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 >> 172.21.15.90:0/426104169 conn(0x7f27a80fb200 msgr2=0x7f27a80fccf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.588+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 shutdown_connections 2024-04-06T14:04:53.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.588+0000 7f27aca07700 1 -- 172.21.15.90:0/426104169 wait complete. 2024-04-06T14:04:53.600 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:04:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:53 smithi134 ceph-mon[47549]: pgmap v245: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:53 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2498089022' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:53 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3057046340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:53 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3251820757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 -- 172.21.15.90:0/2732098219 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc102840 msgr2=0x7f30dc1090f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 --2- 172.21.15.90:0/2732098219 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc102840 0x7f30dc1090f0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f30d400b0d0 tx=0x7f30d400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 -- 172.21.15.90:0/2732098219 shutdown_connections 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 --2- 172.21.15.90:0/2732098219 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc102840 0x7f30dc1090f0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 --2- 172.21.15.90:0/2732098219 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 0x7f30dc1010d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 --2- 172.21.15.90:0/2732098219 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc101ea0 0x7f30dc102300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 -- 172.21.15.90:0/2732098219 >> 172.21.15.90:0/2732098219 conn(0x7f30dc0fc220 msgr2=0x7f30dc0fe660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.724+0000 7f30e4093700 1 -- 172.21.15.90:0/2732098219 shutdown_connections 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 -- 172.21.15.90:0/2732098219 wait complete. 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 Processor -- start 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 -- start start 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 0x7f30dc196cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc102840 0x7f30dc19e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30dc19e7f0 con 0x7f30dc101ea0 2024-04-06T14:04:53.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30dc19e960 con 0x7f30dc102840 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e4093700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f30dc19eb20 con 0x7f30dc100cb0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e162e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e162e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38482/0 (socket says 172.21.15.90:38482) 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.725+0000 7f30e162e700 1 -- 172.21.15.90:0/1507300782 learned_addr learned my addr 172.21.15.90:0/1507300782 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e1e2f700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 0x7f30dc196cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e2630700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc102840 0x7f30dc19e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 -- 172.21.15.90:0/1507300782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 msgr2=0x7f30dc196cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 0x7f30dc196cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 -- 172.21.15.90:0/1507300782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc102840 msgr2=0x7f30dc19e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc102840 0x7f30dc19e220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 -- 172.21.15.90:0/1507300782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30d4009d00 con 0x7f30dc101ea0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e162e700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f30d800c920 tx=0x7f30d800cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30d8007b00 con 0x7f30dc101ea0 2024-04-06T14:04:53.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f30dc19ee00 con 0x7f30dc101ea0 2024-04-06T14:04:53.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f30d8004cf0 con 0x7f30dc101ea0 2024-04-06T14:04:53.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30d8005520 con 0x7f30dc101ea0 2024-04-06T14:04:53.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.726+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f30dc19f290 con 0x7f30dc101ea0 2024-04-06T14:04:53.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.727+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f30dc108060 con 0x7f30dc101ea0 2024-04-06T14:04:53.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.728+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f30d8004750 con 0x7f30dc101ea0 2024-04-06T14:04:53.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.728+0000 7f30d2ffd700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 0x7f30c80a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:53.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.728+0000 7f30e1e2f700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 0x7f30c80a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:53.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.728+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f30d80cc920 con 0x7f30dc101ea0 2024-04-06T14:04:53.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.729+0000 7f30e1e2f700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 0x7f30c80a3660 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f30cc005fd0 tx=0x7f30cc005dd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:53.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.732+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f30d8096100 con 0x7f30dc101ea0 2024-04-06T14:04:53.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:53 smithi105 ceph-mon[46969]: pgmap v245: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:53.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:53 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2498089022' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:53.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:53 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3057046340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:53 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3251820757' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:53.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.929+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f30dc067e70 con 0x7f30dc101ea0 2024-04-06T14:04:53.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.929+0000 7f30d2ffd700 1 -- 172.21.15.90:0/1507300782 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v81) v1 ==== 104+0+11 (secure 0 0 0) 0x7f30d801d0a0 con 0x7f30dc101ea0 2024-04-06T14:04:53.930 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:04:53.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 msgr2=0x7f30c80a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 0x7f30c80a3660 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f30cc005fd0 tx=0x7f30cc005dd0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 msgr2=0x7f30dc197210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f30d800c920 tx=0x7f30d800cce0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 shutdown_connections 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f30c80a11a0 0x7f30c80a3660 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f30dc102840 0x7f30dc19e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f30dc100cb0 0x7f30dc196cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 --2- 172.21.15.90:0/1507300782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f30dc101ea0 0x7f30dc197210 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 >> 172.21.15.90:0/1507300782 conn(0x7f30dc0fc220 msgr2=0x7f30dc1051a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 shutdown_connections 2024-04-06T14:04:53.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:53.932+0000 7f30e4093700 1 -- 172.21.15.90:0/1507300782 wait complete. 2024-04-06T14:04:53.945 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:04:54.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 -- 172.21.15.90:0/3779175572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0100e80 msgr2=0x7f3fa01012e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 --2- 172.21.15.90:0/3779175572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0100e80 0x7f3fa01012e0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f3f940099c0 tx=0x7f3f94009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 -- 172.21.15.90:0/3779175572 shutdown_connections 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 --2- 172.21.15.90:0/3779175572 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0101820 0x7f3fa01080d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 --2- 172.21.15.90:0/3779175572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 0x7f3fa01000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 --2- 172.21.15.90:0/3779175572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0100e80 0x7f3fa01012e0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.068+0000 7f3fa8021700 1 -- 172.21.15.90:0/3779175572 >> 172.21.15.90:0/3779175572 conn(0x7f3fa00fb200 msgr2=0x7f3fa00fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- 172.21.15.90:0/3779175572 shutdown_connections 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- 172.21.15.90:0/3779175572 wait complete. 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 Processor -- start 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- start start 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 0x7f3fa0069b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0100e80 0x7f3fa019b9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3fa019e740 con 0x7f3fa0101820 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3fa019e8b0 con 0x7f3fa00ffc90 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa8021700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3fa019ea70 con 0x7f3fa0100e80 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa65be700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa65be700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38498/0 (socket says 172.21.15.90:38498) 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.069+0000 7f3fa65be700 1 -- 172.21.15.90:0/909291261 learned_addr learned my addr 172.21.15.90:0/909291261 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa55bc700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0100e80 0x7f3fa019b9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa5dbd700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 0x7f3fa0069b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 -- 172.21.15.90:0/909291261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0100e80 msgr2=0x7f3fa019b9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0100e80 0x7f3fa019b9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 -- 172.21.15.90:0/909291261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 msgr2=0x7f3fa0069b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 0x7f3fa0069b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 -- 172.21.15.90:0/909291261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f940095f0 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa65be700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f3f98007d70 tx=0x7f3f9800a340 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f9800ad40 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3fa019ed50 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f980128e0 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f980116b0 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.070+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3fa019f180 con 0x7f3fa0101820 2024-04-06T14:04:54.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.071+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3fa0107040 con 0x7f3fa0101820 2024-04-06T14:04:54.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.073+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f3f98012420 con 0x7f3fa0101820 2024-04-06T14:04:54.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.073+0000 7f3f92ffd700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 0x7f3f8c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.073+0000 7f3fa5dbd700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 0x7f3f8c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.073+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f3f980ccd70 con 0x7f3fa0101820 2024-04-06T14:04:54.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.074+0000 7f3fa5dbd700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 0x7f3f8c0a3670 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0100ce0 tx=0x7f3f9c00b5b0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.077+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3f98096660 con 0x7f3fa0101820 2024-04-06T14:04:54.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:54 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/426104169' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.258 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:54 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1507300782' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.275+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f3fa010b5b0 con 0x7f3fa0101820 2024-04-06T14:04:54.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.275+0000 7f3f92ffd700 1 -- 172.21.15.90:0/909291261 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v81) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3f98099c80 con 0x7f3fa0101820 2024-04-06T14:04:54.276 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:04:54.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 msgr2=0x7f3f8c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 0x7f3f8c0a3670 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f3fa0100ce0 tx=0x7f3f9c00b5b0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 msgr2=0x7f3fa019dfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f3f98007d70 tx=0x7f3f9800a340 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 shutdown_connections 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3f8c0a11b0 0x7f3f8c0a3670 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.277+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3fa0101820 0x7f3fa019dfa0 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.278+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3fa00ffc90 0x7f3fa0069b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.278+0000 7f3fa8021700 1 --2- 172.21.15.90:0/909291261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3fa0100e80 0x7f3fa019b9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.278+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 >> 172.21.15.90:0/909291261 conn(0x7f3fa00fb200 msgr2=0x7f3fa0104180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.278+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 shutdown_connections 2024-04-06T14:04:54.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.278+0000 7f3fa8021700 1 -- 172.21.15.90:0/909291261 wait complete. 2024-04-06T14:04:54.290 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 -- 172.21.15.90:0/1654822708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01040b0 msgr2=0x7f8ff01064a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/1654822708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01040b0 0x7f8ff01064a0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f8fe0009990 tx=0x7f8fe0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 -- 172.21.15.90:0/1654822708 shutdown_connections 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/1654822708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff01069e0 0x7f8ff0108df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/1654822708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff0101780 0x7f8ff0103b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/1654822708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01040b0 0x7f8ff01064a0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 -- 172.21.15.90:0/1654822708 >> 172.21.15.90:0/1654822708 conn(0x7f8ff00fb110 msgr2=0x7f8ff00fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 -- 172.21.15.90:0/1654822708 shutdown_connections 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.411+0000 7f8ff81f4700 1 -- 172.21.15.90:0/1654822708 wait complete. 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 Processor -- start 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 -- start start 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff0101780 0x7f8ff0196c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff01040b0 0x7f8ff0197180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ff019e770 con 0x7f8ff01069e0 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ff019e8b0 con 0x7f8ff01040b0 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff81f4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ff019ea40 con 0x7f8ff0101780 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff6791700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff6791700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38518/0 (socket says 172.21.15.90:38518) 2024-04-06T14:04:54.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff6791700 1 -- 172.21.15.90:0/4123953428 learned_addr learned my addr 172.21.15.90:0/4123953428 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff578f700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff01040b0 0x7f8ff0197180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.412+0000 7f8ff5f90700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff0101780 0x7f8ff0196c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 -- 172.21.15.90:0/4123953428 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff0101780 msgr2=0x7f8ff0196c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff0101780 0x7f8ff0196c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 -- 172.21.15.90:0/4123953428 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff01040b0 msgr2=0x7f8ff0197180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff01040b0 0x7f8ff0197180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 -- 172.21.15.90:0/4123953428 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8fe00095f0 con 0x7f8ff01069e0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff6791700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f8fe800b970 tx=0x7f8fe800bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fe800d860 con 0x7f8ff01069e0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8fe800de80 con 0x7f8ff01069e0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8ff019ed20 con 0x7f8ff01069e0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fe8017400 con 0x7f8ff01069e0 2024-04-06T14:04:54.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.413+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ff019f180 con 0x7f8ff01069e0 2024-04-06T14:04:54.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.414+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8ff00fcc90 con 0x7f8ff01069e0 2024-04-06T14:04:54.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.415+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f8fe800e630 con 0x7f8ff01069e0 2024-04-06T14:04:54.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.415+0000 7f8fe6ffd700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 0x7f8fdc0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.416+0000 7f8ff5f90700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 0x7f8fdc0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.416+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f8fe8006e40 con 0x7f8ff01069e0 2024-04-06T14:04:54.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.416+0000 7f8ff5f90700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 0x7f8fdc0a3670 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f8fec005950 tx=0x7f8fec00b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.420+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8fe8095cd0 con 0x7f8ff01069e0 2024-04-06T14:04:54.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:54 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/426104169' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:54 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1507300782' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.660+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8ff0067190 con 0x7f8ff01069e0 2024-04-06T14:04:54.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.661+0000 7f8fe6ffd700 1 -- 172.21.15.90:0/4123953428 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f8fe80992f0 con 0x7f8ff01069e0 2024-04-06T14:04:54.662 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:54.662 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 msgr2=0x7f8fdc0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 0x7f8fdc0a3670 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f8fec005950 tx=0x7f8fec00b410 comp rx=0 tx=0).stop 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 msgr2=0x7f8ff019e1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f8fe800b970 tx=0x7f8fe800bc80 comp rx=0 tx=0).stop 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 shutdown_connections 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f8fdc0a11b0 0x7f8fdc0a3670 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ff01069e0 0x7f8ff019e1a0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ff0101780 0x7f8ff0196c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 --2- 172.21.15.90:0/4123953428 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ff01040b0 0x7f8ff0197180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 >> 172.21.15.90:0/4123953428 conn(0x7f8ff00fb110 msgr2=0x7f8ff01021c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:54.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 shutdown_connections 2024-04-06T14:04:54.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.663+0000 7f8ff81f4700 1 -- 172.21.15.90:0/4123953428 wait complete. 2024-04-06T14:04:54.665 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:04:54.676 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:04:54.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:54 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/426104169' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:54 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1507300782' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 -- 172.21.15.90:0/3834135399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6640ffba0 msgr2=0x7fb6640fffc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 --2- 172.21.15.90:0/3834135399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6640ffba0 0x7fb6640fffc0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7fb6580099c0 tx=0x7fb658009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 -- 172.21.15.90:0/3834135399 shutdown_connections 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 --2- 172.21.15.90:0/3834135399 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb664101730 0x7fb664107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 --2- 172.21.15.90:0/3834135399 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6640ffba0 0x7fb6640fffc0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 --2- 172.21.15.90:0/3834135399 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664100d90 0x7fb6641011f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.797+0000 7fb669110700 1 -- 172.21.15.90:0/3834135399 >> 172.21.15.90:0/3834135399 conn(0x7fb6640fb110 msgr2=0x7fb6640fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.798+0000 7fb669110700 1 -- 172.21.15.90:0/3834135399 shutdown_connections 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.798+0000 7fb669110700 1 -- 172.21.15.90:0/3834135399 wait complete. 2024-04-06T14:04:54.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.798+0000 7fb669110700 1 Processor -- start 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.798+0000 7fb669110700 1 -- start start 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 0x7fb664069d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 0x7fb66419df60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb66419e750 con 0x7fb664100d90 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb66419e8c0 con 0x7fb664101730 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb66419ea80 con 0x7fb6640ffba0 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38536/0 (socket says 172.21.15.90:38536) 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 -- 172.21.15.90:0/1136009300 learned_addr learned my addr 172.21.15.90:0/1136009300 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:54.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66359e700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 0x7fb66419df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb662d9d700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 0x7fb664069d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 -- 172.21.15.90:0/1136009300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 msgr2=0x7fb664069d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 0x7fb664069d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 -- 172.21.15.90:0/1136009300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 msgr2=0x7fb66419df60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 0x7fb66419df60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 -- 172.21.15.90:0/1136009300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6580095f0 con 0x7fb664100d90 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66359e700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 0x7fb66419df60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb662d9d700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 0x7fb664069d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb66259c700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7fb65400c920 tx=0x7fb65400cce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb654007b00 con 0x7fb664100d90 2024-04-06T14:04:54.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb654004cf0 con 0x7fb664100d90 2024-04-06T14:04:54.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb66419ed00 con 0x7fb664100d90 2024-04-06T14:04:54.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb654005610 con 0x7fb664100d90 2024-04-06T14:04:54.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.799+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb66419f1c0 con 0x7fb664100d90 2024-04-06T14:04:54.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.800+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb664106fc0 con 0x7fb664100d90 2024-04-06T14:04:54.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.801+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fb654004750 con 0x7fb664100d90 2024-04-06T14:04:54.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.802+0000 7fb64bfff700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 0x7fb64c0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:54.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.802+0000 7fb662d9d700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 0x7fb64c0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:54.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.802+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fb6540cc980 con 0x7fb664100d90 2024-04-06T14:04:54.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.802+0000 7fb662d9d700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 0x7fb64c0a36f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fb658006010 tx=0x7fb658009f10 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:54.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:54.805+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb654096060 con 0x7fb664100d90 2024-04-06T14:04:55.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.003+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fb664067e70 con 0x7fb664100d90 2024-04-06T14:04:55.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.004+0000 7fb64bfff700 1 -- 172.21.15.90:0/1136009300 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v81) v1 ==== 74+0+22108 (secure 0 0 0) 0x7fb65401d020 con 0x7fb664100d90 2024-04-06T14:04:55.005 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:55.005 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":81,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:04:02.432114+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:04:55.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 msgr2=0x7fb64c0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 0x7fb64c0a36f0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fb658006010 tx=0x7fb658009f10 comp rx=0 tx=0).stop 2024-04-06T14:04:55.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 msgr2=0x7fb66419b940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7fb65400c920 tx=0x7fb65400cce0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 shutdown_connections 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb64c0a1230 0x7fb64c0a36f0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb664101730 0x7fb66419df60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6640ffba0 0x7fb664069d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 --2- 172.21.15.90:0/1136009300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb664100d90 0x7fb66419b940 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.006+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 >> 172.21.15.90:0/1136009300 conn(0x7fb6640fb110 msgr2=0x7fb664104100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.007+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 shutdown_connections 2024-04-06T14:04:55.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.007+0000 7fb669110700 1 -- 172.21.15.90:0/1136009300 wait complete. 2024-04-06T14:04:55.020 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:04:55.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 -- 172.21.15.90:0/1497895458 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee580ffb80 msgr2=0x7fee580fffa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 --2- 172.21.15.90:0/1497895458 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee580ffb80 0x7fee580fffa0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fee4c0099c0 tx=0x7fee4c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 -- 172.21.15.90:0/1497895458 shutdown_connections 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 --2- 172.21.15.90:0/1497895458 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee58101710 0x7fee58107fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 --2- 172.21.15.90:0/1497895458 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee580ffb80 0x7fee580fffa0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 --2- 172.21.15.90:0/1497895458 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58100d70 0x7fee581011d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 -- 172.21.15.90:0/1497895458 >> 172.21.15.90:0/1497895458 conn(0x7fee580fb0f0 msgr2=0x7fee580fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 -- 172.21.15.90:0/1497895458 shutdown_connections 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.142+0000 7fee5ef63700 1 -- 172.21.15.90:0/1497895458 wait complete. 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 Processor -- start 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 -- start start 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee580ffb80 0x7fee58198e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58101710 0x7fee581a03a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee581a0970 con 0x7fee58100d70 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee581a0ab0 con 0x7fee580ffb80 2024-04-06T14:04:55.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ef63700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee581a0c40 con 0x7fee58101710 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38554/0 (socket says 172.21.15.90:38554) 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 -- 172.21.15.90:0/917103870 learned_addr learned my addr 172.21.15.90:0/917103870 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5ccff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee580ffb80 0x7fee58198e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee5d500700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58101710 0x7fee581a03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 -- 172.21.15.90:0/917103870 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58101710 msgr2=0x7fee581a03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58101710 0x7fee581a03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 -- 172.21.15.90:0/917103870 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee580ffb80 msgr2=0x7fee58198e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee580ffb80 0x7fee58198e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 -- 172.21.15.90:0/917103870 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee4c0095f0 con 0x7fee58100d70 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.143+0000 7fee57fff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fee4400d950 tx=0x7fee4400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.144+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee4400f890 con 0x7fee58100d70 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.144+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fee44010460 con 0x7fee58100d70 2024-04-06T14:04:55.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.144+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee4400e610 con 0x7fee58100d70 2024-04-06T14:04:55.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.144+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fee581a0f20 con 0x7fee58100d70 2024-04-06T14:04:55.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.144+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fee581a13a0 con 0x7fee58100d70 2024-04-06T14:04:55.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.145+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fee58067190 con 0x7fee58100d70 2024-04-06T14:04:55.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.146+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fee440108d0 con 0x7fee58100d70 2024-04-06T14:04:55.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.146+0000 7fee55ffb700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 0x7fee480a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.147+0000 7fee5ccff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 0x7fee480a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.147+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fee440cc5e0 con 0x7fee58100d70 2024-04-06T14:04:55.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.147+0000 7fee5ccff700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 0x7fee480a36f0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fee4c003730 tx=0x7fee4c000bc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.151+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fee44095dc0 con 0x7fee58100d70 2024-04-06T14:04:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:55 smithi090 ceph-mon[43420]: pgmap v246: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:04:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:55 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/909291261' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:55 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4123953428' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:55 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1136009300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.394+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fee5810b390 con 0x7fee58100d70 2024-04-06T14:04:55.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.395+0000 7fee55ffb700 1 -- 172.21.15.90:0/917103870 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fee440993e0 con 0x7fee58100d70 2024-04-06T14:04:55.395 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:04:55.395 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 msgr2=0x7fee480a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 0x7fee480a36f0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fee4c003730 tx=0x7fee4c000bc0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 msgr2=0x7fee58199380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fee4400d950 tx=0x7fee4400dc60 comp rx=0 tx=0).stop 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 shutdown_connections 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fee480a1230 0x7fee480a36f0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fee58101710 0x7fee581a03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fee580ffb80 0x7fee58198e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 --2- 172.21.15.90:0/917103870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fee58100d70 0x7fee58199380 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 >> 172.21.15.90:0/917103870 conn(0x7fee580fb0f0 msgr2=0x7fee580690b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 shutdown_connections 2024-04-06T14:04:55.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.397+0000 7fee5ef63700 1 -- 172.21.15.90:0/917103870 wait complete. 2024-04-06T14:04:55.399 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:04:55.411 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 23, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-04-06T14:04:01.430508+0000', 'modified': '2024-04-06T14:04:06.509384+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24515, 'mds_1': 24490, 'mds_2': 24485}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24457': {'gid': 24457, 'name': 'f', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.134:6835/2168599536', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6834', 'nonce': 2168599536}, {'type': 'v1', 'addr': '172.21.15.134:6835', 'nonce': 2168599536}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24458': {'gid': 24458, 'name': 'e', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.105:6835/3686963284', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6834', 'nonce': 3686963284}, {'type': 'v1', 'addr': '172.21.15.105:6835', 'nonce': 3686963284}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24469': {'gid': 24469, 'name': 'c', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.134:6837/659065731', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6836', 'nonce': 659065731}, {'type': 'v1', 'addr': '172.21.15.134:6837', 'nonce': 659065731}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24485': {'gid': 24485, 'name': 'h', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.105:6837/2851238848', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6836', 'nonce': 2851238848}, {'type': 'v1', 'addr': '172.21.15.105:6837', 'nonce': 2851238848}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24490': {'gid': 24490, 'name': 'i', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.134:6839/1913582469', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6838', 'nonce': 1913582469}, {'type': 'v1', 'addr': '172.21.15.134:6839', 'nonce': 1913582469}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24515': {'gid': 24515, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.105:6839/2253165057', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6838', 'nonce': 2253165057}, {'type': 'v1', 'addr': '172.21.15.105:6839', 'nonce': 2253165057}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0} 2024-04-06T14:04:55.411 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-04-06T14:04:55.411 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2024-04-06T14:04:55.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 -- 172.21.15.90:0/3665127465 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 msgr2=0x7ff8b4106510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 --2- 172.21.15.90:0/3665127465 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4106510 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0009990 tx=0x7ff8b0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 -- 172.21.15.90:0/3665127465 shutdown_connections 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 --2- 172.21.15.90:0/3665127465 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4106a50 0x7ff8b4108ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 --2- 172.21.15.90:0/3665127465 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b41017f0 0x7ff8b4103be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 --2- 172.21.15.90:0/3665127465 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4106510 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 -- 172.21.15.90:0/3665127465 >> 172.21.15.90:0/3665127465 conn(0x7ff8b40fb200 msgr2=0x7ff8b40fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 -- 172.21.15.90:0/3665127465 shutdown_connections 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.533+0000 7ff8bc715700 1 -- 172.21.15.90:0/3665127465 wait complete. 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 Processor -- start 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 -- start start 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b41017f0 0x7ff8b4196d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 0x7ff8b419e250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419e820 con 0x7ff8b4104120 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419e990 con 0x7ff8b41017f0 2024-04-06T14:04:55.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bc715700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419eb50 con 0x7ff8b4106a50 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38582/0 (socket says 172.21.15.90:38582) 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/1775890781 learned_addr learned my addr 172.21.15.90:0/1775890781 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b41017f0 0x7ff8b4196d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 0x7ff8b419e250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/1775890781 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 msgr2=0x7ff8b419e250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 0x7ff8b419e250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/1775890781 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b41017f0 msgr2=0x7ff8b4196d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.534+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b41017f0 0x7ff8b4196d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/1775890781 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8b00095f0 con 0x7ff8b4104120 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 0x7ff8b419e250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7ff8b00095c0 tx=0x7ff8b000f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8ab7fe700 1 -- 172.21.15.90:0/1775890781 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b001d070 con 0x7ff8b4104120 2024-04-06T14:04:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8b419edd0 con 0x7ff8b4104120 2024-04-06T14:04:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8ab7fe700 1 -- 172.21.15.90:0/1775890781 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8b0022470 con 0x7ff8b4104120 2024-04-06T14:04:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8ab7fe700 1 -- 172.21.15.90:0/1775890781 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0017650 con 0x7ff8b4104120 2024-04-06T14:04:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.535+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8b419f1a0 con 0x7ff8b4104120 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.537+0000 7ff8ab7fe700 1 -- 172.21.15.90:0/1775890781 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7ff8b0022850 con 0x7ff8b4104120 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.538+0000 7ff8ab7fe700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 0x7ff8a00a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.538+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 0x7ff8a00a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.538+0000 7ff8ab7fe700 1 -- 172.21.15.90:0/1775890781 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7ff8b00cddc0 con 0x7ff8b4104120 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.538+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 0x7ff8a00a3670 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7ff8b4102fa0 tx=0x7ff8a4009190 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 Processor -- start 2024-04-06T14:04:55.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 -- start start 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604360 0x561389604970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x5613896053b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56138955b940 con 0x561389603a00 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389559300 con 0x561389604f50 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389554bf0 con 0x561389604360 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38590/0 (socket says 172.21.15.90:38590) 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 -- 172.21.15.90:0/2009123941 learned_addr learned my addr 172.21.15.90:0/2009123941 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604360 0x561389604970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x5613896053b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 -- 172.21.15.90:0/2009123941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604360 msgr2=0x561389604970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604360 0x561389604970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 -- 172.21.15.90:0/2009123941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 msgr2=0x5613896053b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x5613896053b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8ba4b1700 1 -- 172.21.15.90:0/2009123941 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5613893d6360 con 0x561389603a00 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.539+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x5613896053b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4009fd0 tx=0x7ff8a4011000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3336e9c9e7ad3be server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8a8ff9700 1 -- 172.21.15.90:0/2009123941 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a400ee90 con 0x561389603a00 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8a8ff9700 1 -- 172.21.15.90:0/2009123941 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a4015940 con 0x561389603a00 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8a8ff9700 1 -- 172.21.15.90:0/2009123941 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a40139b0 con 0x561389603a00 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2009123941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 msgr2=0x561389603e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.540+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4009fd0 tx=0x7ff8a4011000 comp rx=0 tx=0).stop 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2009123941 shutdown_connections 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x5613896053b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389603a00 0x561389603e20 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2009123941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604360 0x561389604970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2009123941 >> 172.21.15.90:0/2009123941 conn(0x561389562650 msgr2=0x5613895fe560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2009123941 shutdown_connections 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2009123941 wait complete. 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- start start 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 Processor -- start 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389603a00 0x56138960d610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604f50 0x561389614b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5613894f0d30 con 0x561389604360 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5613894f09d0 con 0x561389603a00 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5613893d8650 con 0x561389604f50 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8c88c9b80 10 client.0 _ll_register_callbacks cb 0x5613894fa420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38602/0 (socket says 172.21.15.90:38602) 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2556762728 learned_addr learned my addr 172.21.15.90:0/2556762728 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.541+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389603a00 0x56138960d610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604f50 0x561389614b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2556762728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604f50 msgr2=0x561389614b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604f50 0x561389614b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389603a00 msgr2=0x56138960d610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389603a00 0x56138960d610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2556762728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8a400e6c0 con 0x561389604360 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0006010 tx=0x7ff8b009ba60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=acececc9b9aa4d50 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0097040 con 0x561389604360 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x561389515330 con 0x561389604360 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7ff8b009b520 con 0x561389604360 2024-04-06T14:04:55.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff8b009f350 con 0x561389604360 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 10 client.15063 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x561389382b10 con 0x561389604360 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x561389515330 con 0x561389604360 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff8b0097310 con 0x561389604360 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 10 client.15063 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 10 client.15063 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 20 client.15063 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389618430 0x56138961d6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 4 client.15063 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.542+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56138954d6b0 con 0x561389618430 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.543+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389618430 0x56138961d6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.543+0000 7ff89dffb700 10 client.15063 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.543+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389618430 0x56138961d6d0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e1d3cc9928ef2901 server_cookie=83c892ccbb1b6755 in_seq=0 out_seq=0 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff89dffb700 1 -- 172.21.15.90:0/2556762728 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56138954d6b0 con 0x561389618430 2024-04-06T14:04:55.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff89dffb700 10 client.15063 handle_command_reply: tid=0 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 client.15063 shutdown 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 msgr2=0x56138960db50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0006010 tx=0x7ff8b009ba60 comp rx=0 tx=0).stop 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 shutdown_connections 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389618430 0x56138961d6d0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389604f50 0x561389614b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389603a00 0x56138960d610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2556762728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x56138960db50 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 >> 172.21.15.90:0/2556762728 conn(0x561389562650 msgr2=0x5613895fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff89dffb700 0 client.15063 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 shutdown_connections 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2556762728 wait complete. 2024-04-06T14:04:55.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.544+0000 7ff8c88c9b80 20 client.15063 trim_cache size 0 max 16384 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 Processor -- start 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 -- start start 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389615240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x56138960b490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56138954d6e0 con 0x561389604360 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389515360 con 0x561389604f50 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389512ad0 con 0x561389603a00 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38618/0 (socket says 172.21.15.90:38618) 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2766426226 learned_addr learned my addr 172.21.15.90:0/2766426226 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2766426226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 msgr2=0x561389615240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389615240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2766426226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 msgr2=0x56138960b490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x56138960b490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 -- 172.21.15.90:0/2766426226 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56138951ae50 con 0x561389604360 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.562+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0097800 tx=0x7ff8b009f230 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c76e15ff7b177769 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8a97fa700 1 -- 172.21.15.90:0/2766426226 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b0097040 con 0x561389604360 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8a97fa700 1 -- 172.21.15.90:0/2766426226 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8b009b520 con 0x561389604360 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8a97fa700 1 -- 172.21.15.90:0/2766426226 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8b000fad0 con 0x561389604360 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2766426226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 msgr2=0x561389614b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7ff8b0097800 tx=0x7ff8b009f230 comp rx=0 tx=0).stop 2024-04-06T14:04:55.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2766426226 shutdown_connections 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604f50 0x56138960b490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389615240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/2766426226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604360 0x561389614b50 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2766426226 >> 172.21.15.90:0/2766426226 conn(0x561389562650 msgr2=0x5613895fe530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2766426226 shutdown_connections 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/2766426226 wait complete. 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.563+0000 7ff8c88c9b80 1 -- start start 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 Processor -- start 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389614820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 0x5613896112c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389615040 con 0x561389604f50 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389611d70 con 0x561389604360 2024-04-06T14:04:55.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561389611f30 con 0x561389603a00 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 10 client.0 _ll_register_callbacks cb 0x5613894fa420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38626/0 (socket says 172.21.15.90:38626) 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 -- 172.21.15.90:0/1055359316 learned_addr learned my addr 172.21.15.90:0/1055359316 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389614820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 0x5613896112c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 -- 172.21.15.90:0/1055359316 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 msgr2=0x561389614820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389614820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 -- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 msgr2=0x5613896112c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 0x5613896112c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 -- 172.21.15.90:0/1055359316 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561389382b10 con 0x561389604f50 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8b9cb0700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 0x5613896112c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:55.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8bacb2700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7ff8ac009630 tx=0x7ff8ac007ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6435b1c87180ac7e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff89dffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8ac004bb0 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7ff8ac004d10 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.564+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff8a4015940 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff8ac013820 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 10 client.15069 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x5613893dac80 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8ac027020 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff8ac017970 con 0x561389604f50 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 10 client.15069 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 10 client.15069 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 20 client.15069 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389671a80 0x56138966e440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 4 client.15069 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5613893a3ca0 con 0x561389671a80 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.565+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389671a80 0x56138966e440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff89dffb700 10 client.15069 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8ba4b1700 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389671a80 0x56138966e440 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d9f3578e06286919 server_cookie=debbb2326e7510c in_seq=0 out_seq=0 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff89dffb700 1 -- 172.21.15.90:0/1055359316 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x5613893a3ca0 con 0x561389671a80 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff89dffb700 10 client.15069 handle_command_reply: tid=0 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 client.15069 shutdown 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 msgr2=0x561389611800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7ff8ac009630 tx=0x7ff8ac007ba0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 shutdown_connections 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561389671a80 0x56138966e440 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561389604f50 0x561389611800 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561389603a00 0x561389614820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff89dffb700 0 client.15069 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 --2- 172.21.15.90:0/1055359316 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561389604360 0x5613896112c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 >> 172.21.15.90:0/1055359316 conn(0x561389562650 msgr2=0x5613896216f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 shutdown_connections 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 1 -- 172.21.15.90:0/1055359316 wait complete. 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8c88c9b80 20 client.15069 trim_cache size 0 max 16384 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 2, 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24485, 2024-04-06T14:04:55.574 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 50.063482393000001, 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 81, 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 67.129280317999999 2024-04-06T14:04:55.575 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 msgr2=0x7ff8a00a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 0x7ff8a00a3670 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7ff8b4102fa0 tx=0x7ff8a4009190 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 msgr2=0x7ff8b4197240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7ff8b00095c0 tx=0x7ff8b000f690 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.566+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 shutdown_connections 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff8a00a11b0 0x7ff8a00a3670 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b4106a50 0x7ff8b419e250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b41017f0 0x7ff8b4196d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 --2- 172.21.15.90:0/1775890781 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4104120 0x7ff8b4197240 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 >> 172.21.15.90:0/1775890781 conn(0x7ff8b40fb200 msgr2=0x7ff8b4103690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 shutdown_connections 2024-04-06T14:04:55.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.567+0000 7ff8bc715700 1 -- 172.21.15.90:0/1775890781 wait complete. 2024-04-06T14:04:55.579 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-04-06T14:04:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:55 smithi134 ceph-mon[47549]: pgmap v246: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:04:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:55 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/909291261' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:55 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4123953428' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:55 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1136009300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 -- 172.21.15.90:0/1513513737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c101800 msgr2=0x7efe1c1080b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 --2- 172.21.15.90:0/1513513737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c101800 0x7efe1c1080b0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7efe1400b0d0 tx=0x7efe1400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 -- 172.21.15.90:0/1513513737 shutdown_connections 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 --2- 172.21.15.90:0/1513513737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c101800 0x7efe1c1080b0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 --2- 172.21.15.90:0/1513513737 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c0ffc70 0x7efe1c100090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 --2- 172.21.15.90:0/1513513737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 0x7efe1c1012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 -- 172.21.15.90:0/1513513737 >> 172.21.15.90:0/1513513737 conn(0x7efe1c0fb200 msgr2=0x7efe1c0fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.700+0000 7efe23f68700 1 -- 172.21.15.90:0/1513513737 shutdown_connections 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 -- 172.21.15.90:0/1513513737 wait complete. 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 Processor -- start 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 -- start start 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 0x7efe1c1972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c101800 0x7efe1c19e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe1c19e890 con 0x7efe1c0ffc70 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe1c19ea00 con 0x7efe1c100e60 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe23f68700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe1c19ebc0 con 0x7efe1c101800 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe21d04700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe21d04700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38646/0 (socket says 172.21.15.90:38646) 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.701+0000 7efe21d04700 1 -- 172.21.15.90:0/255950143 learned_addr learned my addr 172.21.15.90:0/255950143 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21503700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 0x7efe1c1972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe22505700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c101800 0x7efe1c19e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 -- 172.21.15.90:0/255950143 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c101800 msgr2=0x7efe1c19e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c101800 0x7efe1c19e2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 -- 172.21.15.90:0/255950143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 msgr2=0x7efe1c1972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 0x7efe1c1972e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 -- 172.21.15.90:0/255950143 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe14009d00 con 0x7efe1c0ffc70 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe21d04700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7efe1800c990 tx=0x7efe1800cca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe12ffd700 1 -- 172.21.15.90:0/255950143 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe18007940 con 0x7efe1c0ffc70 2024-04-06T14:04:55.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe12ffd700 1 -- 172.21.15.90:0/255950143 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe1800f450 con 0x7efe1c0ffc70 2024-04-06T14:04:55.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe1c19eea0 con 0x7efe1c0ffc70 2024-04-06T14:04:55.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe12ffd700 1 -- 172.21.15.90:0/255950143 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe18018660 con 0x7efe1c0ffc70 2024-04-06T14:04:55.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.702+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe1c19f2c0 con 0x7efe1c0ffc70 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.704+0000 7efe12ffd700 1 -- 172.21.15.90:0/255950143 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7efe1800f8c0 con 0x7efe1c0ffc70 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.704+0000 7efe12ffd700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 0x7efe080a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.705+0000 7efe21503700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 0x7efe080a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.705+0000 7efe12ffd700 1 -- 172.21.15.90:0/255950143 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7efe180cc750 con 0x7efe1c0ffc70 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.705+0000 7efe21503700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 0x7efe080a3670 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7efe0c005950 tx=0x7efe0c00a300 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.705+0000 7efe3011cb80 1 Processor -- start 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 -- start start 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dea00 0x55df9a7dee20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dff50 0x55df9a7e03b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a736940 con 0x55df9a7df360 2024-04-06T14:04:55.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a734300 con 0x55df9a7dff50 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe3011cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a72fbf0 con 0x55df9a7dea00 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38652/0 (socket says 172.21.15.90:38652) 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 -- 172.21.15.90:0/3685456860 learned_addr learned my addr 172.21.15.90:0/3685456860 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 -- 172.21.15.90:0/3685456860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dea00 msgr2=0x55df9a7dee20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dea00 0x55df9a7dee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 -- 172.21.15.90:0/3685456860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dff50 msgr2=0x55df9a7e03b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dff50 0x55df9a7e03b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 -- 172.21.15.90:0/3685456860 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55df9a5b1360 con 0x55df9a7df360 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe21503700 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7efe0c00fde0 tx=0x7efe0c012000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e718bf0e76f278b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe07fff700 1 -- 172.21.15.90:0/3685456860 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe0c010ac0 con 0x55df9a7df360 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe07fff700 1 -- 172.21.15.90:0/3685456860 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe0c020070 con 0x55df9a7df360 2024-04-06T14:04:55.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.706+0000 7efe07fff700 1 -- 172.21.15.90:0/3685456860 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe0c01b420 con 0x55df9a7df360 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 -- 172.21.15.90:0/3685456860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 msgr2=0x55df9a7df970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7efe0c00fde0 tx=0x7efe0c012000 comp rx=0 tx=0).stop 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 -- 172.21.15.90:0/3685456860 shutdown_connections 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dff50 0x55df9a7e03b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dea00 0x55df9a7dee20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3685456860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7df970 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 -- 172.21.15.90:0/3685456860 >> 172.21.15.90:0/3685456860 conn(0x55df9a73d650 msgr2=0x55df9a7d9560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 -- 172.21.15.90:0/3685456860 shutdown_connections 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.707+0000 7efe3011cb80 1 -- 172.21.15.90:0/3685456860 wait complete. 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 -- start start 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 Processor -- start 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e8730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dff50 0x55df9a7efc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a6cbd30 con 0x55df9a7df360 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a6cb9d0 con 0x55df9a7dea00 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a5b3650 con 0x55df9a7dff50 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe3011cb80 10 client.0 _ll_register_callbacks cb 0x55df9a6d5420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38666/0 (socket says 172.21.15.90:38666) 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 -- 172.21.15.90:0/3870013854 learned_addr learned my addr 172.21.15.90:0/3870013854 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21d04700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e8730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 -- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dff50 msgr2=0x55df9a7efc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dff50 0x55df9a7efc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 -- 172.21.15.90:0/3870013854 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 msgr2=0x55df9a7e8730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e8730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21503700 1 -- 172.21.15.90:0/3870013854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe0c00fa90 con 0x55df9a7df360 2024-04-06T14:04:55.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.708+0000 7efe21d04700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e8730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe21503700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7efe0c010710 tx=0x7efe0c00fec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=57469aa55b8cc476 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe0c029070 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7efe0c010bd0 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55df9a6f0330 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7efe0c010d90 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 10 client.15081 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x55df9a55db10 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55df9a6f0330 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7efe0c004ca0 con 0x55df9a7df360 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 10 client.15081 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 10 client.15081 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 20 client.15081 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a7f31b0 0x55df9a7f8aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 4 client.15081 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.709+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55df9a7286b0 con 0x55df9a7f31b0 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.710+0000 7efe21d04700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a7f31b0 0x55df9a7f8aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.710+0000 7efe21d04700 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a7f31b0 0x55df9a7f8aa0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=30b95f0f713b1e42 server_cookie=d59b4ad6b63312bf in_seq=0 out_seq=0 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.710+0000 7efe057fa700 10 client.15081 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe057fa700 1 -- 172.21.15.90:0/3870013854 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55df9a7286b0 con 0x55df9a7f31b0 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe057fa700 10 client.15081 handle_command_reply: tid=0 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 client.15081 shutdown 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 msgr2=0x55df9a7e8c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7efe0c010710 tx=0x7efe0c00fec0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 shutdown_connections 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a7f31b0 0x55df9a7f8aa0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7dff50 0x55df9a7efc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe057fa700 0 client.15081 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e8730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 --2- 172.21.15.90:0/3870013854 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7df360 0x55df9a7e8c70 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 >> 172.21.15.90:0/3870013854 conn(0x55df9a73d650 msgr2=0x55df9a7d8bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 shutdown_connections 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 1 -- 172.21.15.90:0/3870013854 wait complete. 2024-04-06T14:04:55.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.711+0000 7efe3011cb80 20 client.15081 trim_cache size 0 max 16384 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 Processor -- start 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 -- start start 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e7310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7f0310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a7286b0 con 0x55df9a7dff50 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a6f0330 con 0x55df9a7dea00 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.728+0000 7efe3011cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a728990 con 0x55df9a7df360 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38676/0 (socket says 172.21.15.90:38676) 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 -- 172.21.15.90:0/61183949 learned_addr learned my addr 172.21.15.90:0/61183949 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe21503700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7f0310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe21d04700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e7310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 -- 172.21.15.90:0/61183949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 msgr2=0x55df9a7f0310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7f0310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 -- 172.21.15.90:0/61183949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 msgr2=0x55df9a7e7310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e7310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 -- 172.21.15.90:0/61183949 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55df9a6f5e50 con 0x55df9a7dff50 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe21503700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7f0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe21d04700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e7310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe22505700 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7efe1400b0d0 tx=0x7efe14007630 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7998af64c7e5272b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe10ff9700 1 -- 172.21.15.90:0/61183949 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe1400e070 con 0x55df9a7dff50 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe10ff9700 1 -- 172.21.15.90:0/61183949 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe140092d0 con 0x55df9a7dff50 2024-04-06T14:04:55.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.729+0000 7efe10ff9700 1 -- 172.21.15.90:0/61183949 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe140056e0 con 0x55df9a7dff50 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- 172.21.15.90:0/61183949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 msgr2=0x55df9a7e7c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7efe1400b0d0 tx=0x7efe14007630 comp rx=0 tx=0).stop 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- 172.21.15.90:0/61183949 shutdown_connections 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7e7c40 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7e7310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 --2- 172.21.15.90:0/61183949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7f0310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- 172.21.15.90:0/61183949 >> 172.21.15.90:0/61183949 conn(0x55df9a73d650 msgr2=0x55df9a7e6580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- 172.21.15.90:0/61183949 shutdown_connections 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- 172.21.15.90:0/61183949 wait complete. 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 -- start start 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 Processor -- start 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.730+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7efcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7ec7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a848090 con 0x55df9a7dff50 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a8481d0 con 0x55df9a7dea00 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df9a848340 con 0x55df9a7df360 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 10 client.0 _ll_register_callbacks cb 0x7efe0c00fcb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38684/0 (socket says 172.21.15.90:38684) 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 -- 172.21.15.90:0/1448349509 learned_addr learned my addr 172.21.15.90:0/1448349509 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe21d04700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7efcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe21503700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7ec7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 -- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 msgr2=0x55df9a7ec7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7ec7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 -- 172.21.15.90:0/1448349509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 msgr2=0x55df9a7efcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7efcb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 -- 172.21.15.90:0/1448349509 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe1c108a00 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe21503700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7ec7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe22505700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7efe14006260 tx=0x7efe14005610 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2ca56acd8daaba8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe057fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe1400e070 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7efe14006000 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7efe0c020070 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7efe14034480 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 10 client.15087 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.731+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x55df9a5b5c80 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe0c020070 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7efe14019050 con 0x55df9a7dff50 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 10 client.15087 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 10 client.15087 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 20 client.15087 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a84ce10 0x55df9a849760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 4 client.15087 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55df9a57eca0 con 0x55df9a84ce10 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.732+0000 7efe21d04700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a84ce10 0x55df9a849760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe057fa700 10 client.15087 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe21d04700 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a84ce10 0x55df9a849760 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fed47aa0642f6b6c server_cookie=86e5bf8f8f7223e5 in_seq=0 out_seq=0 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe057fa700 1 -- 172.21.15.90:0/1448349509 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x55df9a57eca0 con 0x55df9a84ce10 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe057fa700 10 client.15087 handle_command_reply: tid=0 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 client.15087 shutdown 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 msgr2=0x55df9a7ecd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7efe14006260 tx=0x7efe14005610 comp rx=0 tx=0).stop 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 shutdown_connections 2024-04-06T14:04:55.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55df9a84ce10 0x55df9a849760 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55df9a7dff50 0x55df9a7ecd80 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55df9a7dea00 0x55df9a7efcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 --2- 172.21.15.90:0/1448349509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55df9a7df360 0x55df9a7ec7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe057fa700 0 client.15087 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 >> 172.21.15.90:0/1448349509 conn(0x55df9a73d650 msgr2=0x55df9a7e6170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 shutdown_connections 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 1 -- 172.21.15.90:0/1448349509 wait complete. 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.733+0000 7efe3011cb80 20 client.15087 trim_cache size 0 max 16384 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:04:55.741 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 1, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24490, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 51.236610433000003, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 81, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 65.575029231000002 2024-04-06T14:04:55.742 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 msgr2=0x7efe080a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 0x7efe080a3670 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7efe0c005950 tx=0x7efe0c00a300 comp rx=0 tx=0).stop 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 msgr2=0x7efe1c196da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7efe1800c990 tx=0x7efe1800cca0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 shutdown_connections 2024-04-06T14:04:55.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7efe080a11b0 0x7efe080a3670 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe1c101800 0x7efe1c19e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe1c0ffc70 0x7efe1c196da0 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 --2- 172.21.15.90:0/255950143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe1c100e60 0x7efe1c1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 >> 172.21.15.90:0/255950143 conn(0x7efe1c0fb200 msgr2=0x7efe1c0fcc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 shutdown_connections 2024-04-06T14:04:55.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.734+0000 7efe23f68700 1 -- 172.21.15.90:0/255950143 wait complete. 2024-04-06T14:04:55.746 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-04-06T14:04:55.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:55 smithi105 ceph-mon[46969]: pgmap v246: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:04:55.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:55 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/909291261' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:04:55.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:55 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4123953428' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:55 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1136009300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- 172.21.15.90:0/3525528682 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec100e80 msgr2=0x7fecec1012e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- 172.21.15.90:0/3525528682 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec100e80 0x7fecec1012e0 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7fece80099c0 tx=0x7fece8009cd0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- 172.21.15.90:0/3525528682 shutdown_connections 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- 172.21.15.90:0/3525528682 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec101820 0x7fecec1080d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- 172.21.15.90:0/3525528682 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec0ffc90 0x7fecec1000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- 172.21.15.90:0/3525528682 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec100e80 0x7fecec1012e0 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- 172.21.15.90:0/3525528682 >> 172.21.15.90:0/3525528682 conn(0x7fecec0fb200 msgr2=0x7fecec0fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- 172.21.15.90:0/3525528682 shutdown_connections 2024-04-06T14:04:55.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- 172.21.15.90:0/3525528682 wait complete. 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 Processor -- start 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- start start 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 0x7fecec196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 0x7fecec1972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fecec19e840 con 0x7fecec101820 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fecec19e9b0 con 0x7fecec100e80 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.869+0000 7fecf4414700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fecec19eb70 con 0x7fecec0ffc90 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38700/0 (socket says 172.21.15.90:38700) 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 -- 172.21.15.90:0/818628466 learned_addr learned my addr 172.21.15.90:0/818628466 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf21b0700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 0x7fecec196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf19af700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 0x7fecec1972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 -- 172.21.15.90:0/818628466 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 msgr2=0x7fecec196da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 0x7fecec196da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 -- 172.21.15.90:0/818628466 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 msgr2=0x7fecec1972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 0x7fecec1972e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 -- 172.21.15.90:0/818628466 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fece80095f0 con 0x7fecec101820 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf19af700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 0x7fecec1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf21b0700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 0x7fecec196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf29b1700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fece400b970 tx=0x7fece400bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fece37fe700 1 -- 172.21.15.90:0/818628466 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece400c9b0 con 0x7fecec101820 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fece37fe700 1 -- 172.21.15.90:0/818628466 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fece4014440 con 0x7fecec101820 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fecec19ee50 con 0x7fecec101820 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fece37fe700 1 -- 172.21.15.90:0/818628466 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece40075d0 con 0x7fecec101820 2024-04-06T14:04:55.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.870+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fecec19f220 con 0x7fecec101820 2024-04-06T14:04:55.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.872+0000 7fece37fe700 1 -- 172.21.15.90:0/818628466 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fece40145b0 con 0x7fecec101820 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.873+0000 7fece37fe700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 0x7fecd80a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.873+0000 7fecf21b0700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 0x7fecd80a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.873+0000 7fece37fe700 1 -- 172.21.15.90:0/818628466 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fece4006e40 con 0x7fecec101820 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.873+0000 7fecf21b0700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 0x7fecd80a3670 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fecdc005fd0 tx=0x7fecdc005dd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 Processor -- start 2024-04-06T14:04:55.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 -- start start 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365b9e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365baf50 0x5643365bb3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564336511940 con 0x5643365ba360 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56433650f300 con 0x5643365b9a00 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fed005c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56433650abf0 con 0x5643365baf50 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fecf19af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fecf19af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38706/0 (socket says 172.21.15.90:38706) 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fecf19af700 1 -- 172.21.15.90:0/714998222 learned_addr learned my addr 172.21.15.90:0/714998222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.874+0000 7fecf21b0700 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365b9e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 -- 172.21.15.90:0/714998222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365baf50 msgr2=0x5643365bb3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365baf50 0x5643365bb3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 -- 172.21.15.90:0/714998222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 msgr2=0x5643365b9e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365b9e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 -- 172.21.15.90:0/714998222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56433638c360 con 0x5643365ba360 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf21b0700 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365b9e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fecf19af700 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fece8005f00 tx=0x7fece8004500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e2512a8f7441349d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fece0da5700 1 -- 172.21.15.90:0/714998222 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece800ba50 con 0x5643365ba360 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fece0da5700 1 -- 172.21.15.90:0/714998222 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fece801d040 con 0x5643365ba360 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.875+0000 7fece0da5700 1 -- 172.21.15.90:0/714998222 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece8005300 con 0x5643365ba360 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- 172.21.15.90:0/714998222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 msgr2=0x5643365ba970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fece8005f00 tx=0x7fece8004500 comp rx=0 tx=0).stop 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- 172.21.15.90:0/714998222 shutdown_connections 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365baf50 0x5643365bb3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365b9e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 --2- 172.21.15.90:0/714998222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365ba360 0x5643365ba970 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- 172.21.15.90:0/714998222 >> 172.21.15.90:0/714998222 conn(0x564336518650 msgr2=0x5643365b4560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- 172.21.15.90:0/714998222 shutdown_connections 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- 172.21.15.90:0/714998222 wait complete. 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 -- start start 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.876+0000 7fed005c8b80 1 Processor -- start 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365b9a00 0x5643365c3640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365ba360 0x5643365c3b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643364a6d30 con 0x5643365baf50 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643364a69d0 con 0x5643365ba360 2024-04-06T14:04:55.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56433638e650 con 0x5643365b9a00 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 10 client.0 _ll_register_callbacks cb 0x5643364b0420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38714/0 (socket says 172.21.15.90:38714) 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 -- 172.21.15.90:0/2464410519 learned_addr learned my addr 172.21.15.90:0/2464410519 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf21b0700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365b9a00 0x5643365c3640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 -- 172.21.15.90:0/2464410519 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365b9a00 msgr2=0x5643365c3640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365b9a00 0x5643365c3640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 -- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365ba360 msgr2=0x5643365c3b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf19af700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365ba360 0x5643365c3b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365ba360 0x5643365c3b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 -- 172.21.15.90:0/2464410519 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fecec108a20 con 0x5643365baf50 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecf29b1700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fece4099470 tx=0x7fece4099a20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dcbf3c592b2cb14b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecd5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece409d150 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fece409d310 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5643364cb330 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7fece40a1960 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 10 client.15099 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.877+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x564336338b10 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5643364cb330 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fece40952d0 con 0x5643365baf50 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 10 client.15099 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 10 client.15099 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 20 client.15099 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5643365ce830 0x5643365d3ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 4 client.15099 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:04:55.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5643365036b0 con 0x5643365ce830 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fecf21b0700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5643365ce830 0x5643365d3ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.878+0000 7fecf21b0700 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5643365ce830 0x5643365d3ad0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bdb1d931578fec94 server_cookie=b72927c9814465d0 in_seq=0 out_seq=0 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fecd5ffb700 10 client.15099 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fecd5ffb700 1 -- 172.21.15.90:0/2464410519 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5643365036b0 con 0x5643365ce830 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fecd5ffb700 10 client.15099 handle_command_reply: tid=0 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 client.15099 shutdown 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 msgr2=0x5643365cabc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7fece4099470 tx=0x7fece4099a20 comp rx=0 tx=0).stop 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 shutdown_connections 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cabc0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5643365ce830 0x5643365d3ad0 crc :-1 s=READY pgs=16 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365b9a00 0x5643365c3640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fecd5ffb700 0 client.15099 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:04:55.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 --2- 172.21.15.90:0/2464410519 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365ba360 0x5643365c3b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.879+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 >> 172.21.15.90:0/2464410519 conn(0x564336518650 msgr2=0x5643365b3bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.880+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 shutdown_connections 2024-04-06T14:04:55.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.880+0000 7fed005c8b80 1 -- 172.21.15.90:0/2464410519 wait complete. 2024-04-06T14:04:55.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.880+0000 7fed005c8b80 20 client.15099 trim_cache size 0 max 16384 2024-04-06T14:04:55.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 Processor -- start 2024-04-06T14:04:55.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 -- start start 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365bb390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365b9de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643365c4590 con 0x5643365baf50 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643364c8ad0 con 0x5643365b9a00 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fed005c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564336503990 con 0x5643365ba360 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fecf29b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38728/0 (socket says 172.21.15.90:38728) 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fecf29b1700 1 -- 172.21.15.90:0/3642544847 learned_addr learned my addr 172.21.15.90:0/3642544847 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.897+0000 7fecf21b0700 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365bb390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 -- 172.21.15.90:0/3642544847 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 msgr2=0x5643365b9de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365b9de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 -- 172.21.15.90:0/3642544847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 msgr2=0x5643365bb390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365bb390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 -- 172.21.15.90:0/3642544847 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5643364d0e50 con 0x5643365baf50 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fecf29b1700 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fece400b940 tx=0x7fece4099be0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ff5c4a468ab3a599 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fece15a6700 1 -- 172.21.15.90:0/3642544847 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece409d150 con 0x5643365baf50 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fece15a6700 1 -- 172.21.15.90:0/3642544847 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fece40098b0 con 0x5643365baf50 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fece15a6700 1 -- 172.21.15.90:0/3642544847 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fece40b7ad0 con 0x5643365baf50 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 -- 172.21.15.90:0/3642544847 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 msgr2=0x5643365cb270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fece400b940 tx=0x7fece4099be0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 -- 172.21.15.90:0/3642544847 shutdown_connections 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365baf50 0x5643365cb270 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365b9a00 0x5643365bb390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3642544847 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365b9de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 -- 172.21.15.90:0/3642544847 >> 172.21.15.90:0/3642544847 conn(0x564336518650 msgr2=0x5643365b4530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 -- 172.21.15.90:0/3642544847 shutdown_connections 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.898+0000 7fed005c8b80 1 -- 172.21.15.90:0/3642544847 wait complete. 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 -- start start 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 Processor -- start 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365c75c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365baf50 0x5643365c7b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643365c8120 con 0x5643365b9a00 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643366230c0 con 0x5643365baf50 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564336623200 con 0x5643365ba360 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fed005c8b80 10 client.0 _ll_register_callbacks cb 0x5643364b0420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38736/0 (socket says 172.21.15.90:38736) 2024-04-06T14:04:55.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 -- 172.21.15.90:0/3661184146 learned_addr learned my addr 172.21.15.90:0/3661184146 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf29b1700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365baf50 0x5643365c7b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 -- 172.21.15.90:0/3661184146 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 msgr2=0x5643365c75c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf19af700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365c75c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365c75c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 -- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365baf50 msgr2=0x5643365c7b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365baf50 0x5643365c7b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf21b0700 1 -- 172.21.15.90:0/3661184146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564336338b10 con 0x5643365b9a00 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.899+0000 7fecf19af700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365c75c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecf21b0700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fecec1981d0 tx=0x7fecdc0047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80b350dd7638c670 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fecdc024040 con 0x5643365b9a00 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fecdc027a80 con 0x5643365b9a00 2024-04-06T14:04:55.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564336359ca0 con 0x5643365b9a00 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7fecdc0275c0 con 0x5643365b9a00 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 10 client.15105 fetch_fsmap learned FSMap version 23 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x564336390c80 con 0x5643365b9a00 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564336359ca0 con 0x5643365b9a00 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fecdc01f440 con 0x5643365b9a00 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 10 client.15105 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 10 client.15105 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 20 client.15105 populate_metadata read hostname 'smithi090' 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564336628020 0x564336624970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 4 client.15105 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "status"}) v1 -- 0x56433638ec90 con 0x564336628020 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.900+0000 7fecf19af700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564336628020 0x564336624970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fecd5ffb700 10 client.15105 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:04:55.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fecf19af700 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564336628020 0x564336624970 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ea91ff7183eeae0e server_cookie=4449e58d8dd551c in_seq=0 out_seq=0 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fecd5ffb700 1 -- 172.21.15.90:0/3661184146 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x56433638ec90 con 0x564336628020 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fecd5ffb700 10 client.15105 handle_command_reply: tid=0 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 client.15105 shutdown 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 msgr2=0x5643365caa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7fecec1981d0 tx=0x7fecdc0047c0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 shutdown_connections 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5643365baf50 0x5643365c7b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5643365b9a00 0x5643365caa80 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564336628020 0x564336624970 crc :-1 s=READY pgs=17 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 --2- 172.21.15.90:0/3661184146 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5643365ba360 0x5643365c75c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fecd5ffb700 0 client.15105 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 >> 172.21.15.90:0/3661184146 conn(0x564336518650 msgr2=0x5643365d7af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 shutdown_connections 2024-04-06T14:04:55.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 1 -- 172.21.15.90:0/3661184146 wait complete. 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.901+0000 7fed005c8b80 20 client.15105 trim_cache size 0 max 16384 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 0, 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24515, 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:04:55.910 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 54.468628526000003, 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 81, 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 60.473451406000002 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:04:55.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 msgr2=0x7fecd80a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 0x7fecd80a3670 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fecdc005fd0 tx=0x7fecdc005dd0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 msgr2=0x7fecec19e270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fece400b970 tx=0x7fece400bd30 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 shutdown_connections 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fecd80a11b0 0x7fecd80a3670 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fecec101820 0x7fecec19e270 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fecec0ffc90 0x7fecec196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 --2- 172.21.15.90:0/818628466 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fecec100e80 0x7fecec1972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 >> 172.21.15.90:0/818628466 conn(0x7fecec0fb200 msgr2=0x7fecec0fcd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:55.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 shutdown_connections 2024-04-06T14:04:55.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:55.902+0000 7fecf4414700 1 -- 172.21.15.90:0/818628466 wait complete. 2024-04-06T14:04:55.914 INFO:tasks.cephadm:Setup complete, yielding 2024-04-06T14:04:55.915 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-06T14:04:55.924 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:04:55.925 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph orch status' 2024-04-06T14:04:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:56.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:56 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/917103870' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:56.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:56 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/917103870' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:56.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:56 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/917103870' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 -- 172.21.15.90:0/1046823438 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1018a0 msgr2=0x7f9adc103c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 --2- 172.21.15.90:0/1046823438 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1018a0 0x7f9adc103c90 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f9ad0009990 tx=0x7f9ad0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 -- 172.21.15.90:0/1046823438 shutdown_connections 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 --2- 172.21.15.90:0/1046823438 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc106b00 0x7f9adc108f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 --2- 172.21.15.90:0/1046823438 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1018a0 0x7f9adc103c90 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 --2- 172.21.15.90:0/1046823438 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc1041d0 0x7f9adc1065c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 -- 172.21.15.90:0/1046823438 >> 172.21.15.90:0/1046823438 conn(0x7f9adc0fb2d0 msgr2=0x7f9adc0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:57.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 -- 172.21.15.90:0/1046823438 shutdown_connections 2024-04-06T14:04:57.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.305+0000 7f9ae4350700 1 -- 172.21.15.90:0/1046823438 wait complete. 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 Processor -- start 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 -- start start 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc1018a0 0x7f9adc196d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 0x7f9adc19e2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9adc19e880 con 0x7f9adc1041d0 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9adc19e9c0 con 0x7f9adc1018a0 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae4350700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9adc19eb80 con 0x7f9adc106b00 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae18eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae18eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38756/0 (socket says 172.21.15.90:38756) 2024-04-06T14:04:57.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae18eb700 1 -- 172.21.15.90:0/3473920865 learned_addr learned my addr 172.21.15.90:0/3473920865 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.306+0000 7f9ae20ec700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc1018a0 0x7f9adc196d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae28ed700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 0x7f9adc19e2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 -- 172.21.15.90:0/3473920865 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 msgr2=0x7f9adc19e2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 0x7f9adc19e2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 -- 172.21.15.90:0/3473920865 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc1018a0 msgr2=0x7f9adc196d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc1018a0 0x7f9adc196d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 -- 172.21.15.90:0/3473920865 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ad00095f0 con 0x7f9adc1041d0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae28ed700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 0x7f9adc19e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae18eb700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f9ad800d980 tx=0x7f9ad800dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ad8004a30 con 0x7f9adc1041d0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ad80054e0 con 0x7f9adc1041d0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9adc19ee60 con 0x7f9adc1041d0 2024-04-06T14:04:57.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ad8009c90 con 0x7f9adc1041d0 2024-04-06T14:04:57.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.307+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9adc19f270 con 0x7f9adc1041d0 2024-04-06T14:04:57.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.308+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9adc0fce50 con 0x7f9adc1041d0 2024-04-06T14:04:57.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.309+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9ad8004c20 con 0x7f9adc1041d0 2024-04-06T14:04:57.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.309+0000 7f9acf7fe700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 0x7f9ac80a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:57.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.310+0000 7f9ae20ec700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 0x7f9ac80a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:57.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.310+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f9ad80ccb60 con 0x7f9adc1041d0 2024-04-06T14:04:57.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.310+0000 7f9ae20ec700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 0x7f9ac80a36f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9ad000b5c0 tx=0x7f9ad0003790 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:57.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.313+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9ad80963c0 con 0x7f9adc1041d0 2024-04-06T14:04:57.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.527+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) v1 -- 0x7f9adc02cd20 con 0x7f9ac80a1230 2024-04-06T14:04:57.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.528+0000 7f9acf7fe700 1 -- 172.21.15.90:0/3473920865 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+43 (secure 0 0 0) 0x7f9adc02cd20 con 0x7f9ac80a1230 2024-04-06T14:04:57.529 INFO:teuthology.orchestra.run.smithi090.stdout:Backend: cephadm 2024-04-06T14:04:57.529 INFO:teuthology.orchestra.run.smithi090.stdout:Available: Yes 2024-04-06T14:04:57.529 INFO:teuthology.orchestra.run.smithi090.stdout:Paused: No 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 msgr2=0x7f9ac80a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 0x7f9ac80a36f0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9ad000b5c0 tx=0x7f9ad0003790 comp rx=0 tx=0).stop 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 msgr2=0x7f9adc1972c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f9ad800d980 tx=0x7f9ad800dc90 comp rx=0 tx=0).stop 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 shutdown_connections 2024-04-06T14:04:57.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.530+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9ac80a1230 0x7f9ac80a36f0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9adc106b00 0x7f9adc19e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9adc1018a0 0x7f9adc196d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 --2- 172.21.15.90:0/3473920865 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9adc1041d0 0x7f9adc1972c0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 >> 172.21.15.90:0/3473920865 conn(0x7f9adc0fb2d0 msgr2=0x7f9adc103760 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 shutdown_connections 2024-04-06T14:04:57.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:57.531+0000 7f9ae4350700 1 -- 172.21.15.90:0/3473920865 wait complete. 2024-04-06T14:04:57.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:57 smithi090 ceph-mon[43420]: pgmap v247: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:57.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:57 smithi134 ceph-mon[47549]: pgmap v247: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:57.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:57 smithi105 ceph-mon[46969]: pgmap v247: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:04:57.915 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph orch ps' 2024-04-06T14:04:58.137 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:04:58.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:58 smithi090 ceph-mon[43420]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:58.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:58 smithi090 ceph-mon[43420]: pgmap v248: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-06T14:04:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:58 smithi134 ceph-mon[47549]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:58 smithi134 ceph-mon[47549]: pgmap v248: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-06T14:04:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:58 smithi105 ceph-mon[46969]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:04:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:58 smithi105 ceph-mon[46969]: pgmap v248: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 5 op/s 2024-04-06T14:04:59.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 -- 172.21.15.90:0/3070081743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc801018b0 msgr2=0x7fbc80108160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:59.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 --2- 172.21.15.90:0/3070081743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc801018b0 0x7fbc80108160 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fbc7800b0d0 tx=0x7fbc7800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 -- 172.21.15.90:0/3070081743 shutdown_connections 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 --2- 172.21.15.90:0/3070081743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc801018b0 0x7fbc80108160 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 --2- 172.21.15.90:0/3070081743 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc800ffd10 0x7fbc80100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 --2- 172.21.15.90:0/3070081743 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 0x7fbc80101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 -- 172.21.15.90:0/3070081743 >> 172.21.15.90:0/3070081743 conn(0x7fbc800fb2b0 msgr2=0x7fbc800fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 -- 172.21.15.90:0/3070081743 shutdown_connections 2024-04-06T14:04:59.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.508+0000 7fbc880cd700 1 -- 172.21.15.90:0/3070081743 wait complete. 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.509+0000 7fbc880cd700 1 Processor -- start 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.509+0000 7fbc880cd700 1 -- start start 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.509+0000 7fbc880cd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.509+0000 7fbc880cd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 0x7fbc8019bb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc801018b0 0x7fbc8019e120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc8019e8c0 con 0x7fbc800ffd10 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc8019ea30 con 0x7fbc801018b0 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc8019ebf0 con 0x7fbc80100f10 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38770/0 (socket says 172.21.15.90:38770) 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 -- 172.21.15.90:0/2689608154 learned_addr learned my addr 172.21.15.90:0/2689608154 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:04:59.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85668700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 0x7fbc8019bb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc8666a700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc801018b0 0x7fbc8019e120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 -- 172.21.15.90:0/2689608154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 msgr2=0x7fbc8019bb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 0x7fbc8019bb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 -- 172.21.15.90:0/2689608154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc801018b0 msgr2=0x7fbc8019e120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc801018b0 0x7fbc8019e120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 -- 172.21.15.90:0/2689608154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc78009d00 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc85e69700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fbc7c00e970 tx=0x7fbc7c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc7c009790 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc7c00eec0 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc8019eed0 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc7c003cf0 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.510+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc8019f2f0 con 0x7fbc800ffd10 2024-04-06T14:04:59.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.511+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc80107120 con 0x7fbc800ffd10 2024-04-06T14:04:59.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.513+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fbc7c00ccd0 con 0x7fbc800ffd10 2024-04-06T14:04:59.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.513+0000 7fbc76ffd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 0x7fbc6c0a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:04:59.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.513+0000 7fbc85668700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 0x7fbc6c0a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:04:59.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.513+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fbc7c014070 con 0x7fbc800ffd10 2024-04-06T14:04:59.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.514+0000 7fbc85668700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 0x7fbc6c0a36f0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fbc70005950 tx=0x7fbc7000b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:04:59.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.516+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbc7c0963e0 con 0x7fbc800ffd10 2024-04-06T14:04:59.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.730+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7fbc8002ce20 con 0x7fbc6c0a1230 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.744+0000 7fbc76ffd700 1 -- 172.21.15.90:0/2689608154 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+4185 (secure 0 0 0) 0x7fbc8002ce20 con 0x7fbc6c0a1230 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.a smithi090 running (66s) 61s ago 66s 13.2M - 18.2.2-1086-g052adcad 87261d5ece5a 9cd742b0e160 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.b smithi105 running (64s) 0s ago 64s 29.5M - 18.2.2-1086-g052adcad 87261d5ece5a 5abc943e0837 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.c smithi134 running (73s) 0s ago 73s 19.4M - 18.2.2-1086-g052adcad 87261d5ece5a 45987d90b838 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.d smithi090 running (78s) 61s ago 78s 15.7M - 18.2.2-1086-g052adcad 87261d5ece5a 2898b7aa9e44 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.e smithi105 running (76s) 0s ago 76s 18.3M - 18.2.2-1086-g052adcad 87261d5ece5a f1cfd968c501 2024-04-06T14:04:59.745 INFO:teuthology.orchestra.run.smithi090.stdout:mds.f smithi134 running (75s) 0s ago 74s 19.7M - 18.2.2-1086-g052adcad 87261d5ece5a 9dc0a17e0ad6 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.g smithi090 running (68s) 61s ago 67s 14.2M - 18.2.2-1086-g052adcad 87261d5ece5a fff61848e689 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.h smithi105 running (71s) 0s ago 71s 18.3M - 18.2.2-1086-g052adcad 87261d5ece5a 08ceb974d0ed 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.i smithi134 running (69s) 0s ago 69s 21.8M - 18.2.2-1086-g052adcad 87261d5ece5a 04b8b17f22da 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.j smithi090 running (80s) 61s ago 80s 14.2M - 18.2.2-1086-g052adcad 87261d5ece5a 6e9b4f7e3781 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.k smithi105 running (82s) 0s ago 81s 17.5M - 18.2.2-1086-g052adcad 87261d5ece5a 841e9e3f2b31 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mds.l smithi134 running (84s) 0s ago 83s 14.9M - 18.2.2-1086-g052adcad 87261d5ece5a fd3906325ed3 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.x smithi090 *:9283,8765 running (6m) 61s ago 6m 522M - 18.2.2-1086-g052adcad 87261d5ece5a 1fc7301132dc 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.y smithi105 *:8443,8765 running (5m) 0s ago 5m 459M - 18.2.2-1086-g052adcad 87261d5ece5a 3b87c0798ca9 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mgr.z smithi134 *:8443,8765 running (5m) 0s ago 5m 456M - 18.2.2-1086-g052adcad 87261d5ece5a d5bab1fede19 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mon.a smithi090 running (6m) 61s ago 6m 81.4M 2048M 18.2.2-1086-g052adcad 87261d5ece5a 5fa712d37351 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mon.b smithi105 running (5m) 0s ago 5m 61.1M 2048M 18.2.2-1086-g052adcad 87261d5ece5a bfc0fd3b3b00 2024-04-06T14:04:59.746 INFO:teuthology.orchestra.run.smithi090.stdout:mon.c smithi134 running (5m) 0s ago 5m 63.0M 2048M 18.2.2-1086-g052adcad 87261d5ece5a c4f1b50895df 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 smithi090 running (4m) 61s ago 4m 57.6M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 41f341dcbfba 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 smithi105 running (4m) 0s ago 4m 73.7M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 11e601ae97ba 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 smithi134 running (4m) 0s ago 4m 69.1M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 9f5f74fbc786 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 smithi090 running (3m) 61s ago 3m 120M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 9dd5b0f61cc2 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 smithi105 running (3m) 0s ago 3m 57.8M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 9110e2aaaa0e 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 smithi134 running (3m) 0s ago 3m 131M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 01ce519af36b 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 smithi090 running (3m) 61s ago 3m 116M 4278M 18.2.2-1086-g052adcad 87261d5ece5a d3b15d857b54 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 smithi105 running (2m) 0s ago 2m 129M 4278M 18.2.2-1086-g052adcad 87261d5ece5a f1cc20a689dc 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.8 smithi134 running (2m) 0s ago 2m 60.5M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 642c0d43de88 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.9 smithi090 running (2m) 61s ago 2m 44.3M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 196a935e3fb2 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.10 smithi105 running (117s) 0s ago 117s 59.9M 4278M 18.2.2-1086-g052adcad 87261d5ece5a 0ef47d8f85a9 2024-04-06T14:04:59.747 INFO:teuthology.orchestra.run.smithi090.stdout:osd.11 smithi134 running (100s) 0s ago 100s 119M 4278M 18.2.2-1086-g052adcad 87261d5ece5a ea814de103d5 2024-04-06T14:04:59.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 msgr2=0x7fbc6c0a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 0x7fbc6c0a36f0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fbc70005950 tx=0x7fbc7000b410 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 msgr2=0x7fbc80069290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fbc7c00e970 tx=0x7fbc7c00ec80 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 shutdown_connections 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fbc6c0a1230 0x7fbc6c0a36f0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc801018b0 0x7fbc8019e120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc800ffd10 0x7fbc80069290 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 --2- 172.21.15.90:0/2689608154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc80100f10 0x7fbc8019bb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 >> 172.21.15.90:0/2689608154 conn(0x7fbc800fb2b0 msgr2=0x7fbc80104260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 shutdown_connections 2024-04-06T14:04:59.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:04:59.746+0000 7fbc880cd700 1 -- 172.21.15.90:0/2689608154 wait complete. 2024-04-06T14:04:59.772 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:04:59 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:04:59.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:04:59 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:04:59.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:04:59 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:00.482 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph orch ls' 2024-04-06T14:05:00.733 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:00.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:00 smithi090 ceph-mon[43420]: pgmap v249: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:00 smithi105 ceph-mon[46969]: pgmap v249: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: from='client.15117 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:00 smithi134 ceph-mon[47549]: pgmap v249: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:02.029 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.030 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.030 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:02.030 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:02.030 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:01 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:01 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 -- 172.21.15.90:0/3507836102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 msgr2=0x7f7ddc1023f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 --2- 172.21.15.90:0/3507836102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc1023f0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f7dcc009990 tx=0x7f7dcc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 -- 172.21.15.90:0/3507836102 shutdown_connections 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 --2- 172.21.15.90:0/3507836102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc102930 0x7f7ddc109250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 --2- 172.21.15.90:0/3507836102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc100e10 0x7f7ddc101230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 --2- 172.21.15.90:0/3507836102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc1023f0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 -- 172.21.15.90:0/3507836102 >> 172.21.15.90:0/3507836102 conn(0x7f7ddc0fc3b0 msgr2=0x7f7ddc0fe7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:02.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 -- 172.21.15.90:0/3507836102 shutdown_connections 2024-04-06T14:05:02.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.151+0000 7f7de12bb700 1 -- 172.21.15.90:0/3507836102 wait complete. 2024-04-06T14:05:02.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 Processor -- start 2024-04-06T14:05:02.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 -- start start 2024-04-06T14:05:02.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 0x7f7ddc074b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc102930 0x7f7ddc0736e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ddc073e80 con 0x7f7ddc101f70 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ddc073ff0 con 0x7f7ddc100e10 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7de12bb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7ddc1a0df0 con 0x7f7ddc102930 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7dda7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7dda7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:38802/0 (socket says 172.21.15.90:38802) 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7dda7fc700 1 -- 172.21.15.90:0/44614321 learned_addr learned my addr 172.21.15.90:0/44614321 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7ddaffd700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 0x7f7ddc074b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.152+0000 7f7ddb7fe700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc102930 0x7f7ddc0736e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 -- 172.21.15.90:0/44614321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc102930 msgr2=0x7f7ddc0736e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc102930 0x7f7ddc0736e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 -- 172.21.15.90:0/44614321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 msgr2=0x7f7ddc074b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 0x7f7ddc074b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 -- 172.21.15.90:0/44614321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7dcc0095f0 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7ddaffd700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 0x7f7ddc074b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dda7fc700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f7dcc005910 tx=0x7f7dcc009510 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dcc01d070 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7ddc1a1020 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dcc003fe0 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dcc022dd0 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.153+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ddc1a1490 con 0x7f7ddc101f70 2024-04-06T14:05:02.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.154+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7ddc1081e0 con 0x7f7ddc101f70 2024-04-06T14:05:02.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.155+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f7dcc022470 con 0x7f7ddc101f70 2024-04-06T14:05:02.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.156+0000 7f7dc3fff700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 0x7f7dc40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:02.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.156+0000 7f7ddaffd700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 0x7f7dc40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:02.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.156+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f7dcc0ce4a0 con 0x7f7ddc101f70 2024-04-06T14:05:02.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.156+0000 7f7ddaffd700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 0x7f7dc40a3670 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f7dd0005950 tx=0x7f7dd00058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:02.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.160+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7dcc097d10 con 0x7f7ddc101f70 2024-04-06T14:05:02.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:02.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:01 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' 2024-04-06T14:05:02.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.359+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) v1 -- 0x7f7ddc02cd20 con 0x7f7dc40a11b0 2024-04-06T14:05:02.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.364+0000 7f7dc3fff700 1 -- 172.21.15.90:0/44614321 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+980 (secure 0 0 0) 0x7f7ddc02cd20 con 0x7f7dc40a11b0 2024-04-06T14:05:02.365 INFO:teuthology.orchestra.run.smithi090.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-06T14:05:02.365 INFO:teuthology.orchestra.run.smithi090.stdout:mds.all 12/12 2s ago 88s smithi090=a;smithi090=d;smithi090=g;smithi090=j;smithi105=b;smithi105=e;smithi105=h;smithi105=k;smithi134=c;smithi134=f;smithi134=i;smithi134=l;count:12 2024-04-06T14:05:02.365 INFO:teuthology.orchestra.run.smithi090.stdout:mgr 3/3 2s ago 5m smithi090=x;smithi105=y;smithi134=z;count:3 2024-04-06T14:05:02.366 INFO:teuthology.orchestra.run.smithi090.stdout:mon 3/3 2s ago 5m smithi090:172.21.15.90=a;smithi105:172.21.15.105=b;smithi134:172.21.15.134=c;count:3 2024-04-06T14:05:02.366 INFO:teuthology.orchestra.run.smithi090.stdout:osd 12 2s ago - 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 msgr2=0x7f7dc40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 0x7f7dc40a3670 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f7dd0005950 tx=0x7f7dd00058e0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 msgr2=0x7f7ddc0731a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f7dcc005910 tx=0x7f7dcc009510 comp rx=0 tx=0).stop 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 shutdown_connections 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7dc40a11b0 0x7f7dc40a3670 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7ddc102930 0x7f7ddc0736e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7ddc100e10 0x7f7ddc074b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 --2- 172.21.15.90:0/44614321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7ddc101f70 0x7f7ddc0731a0 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:02.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 >> 172.21.15.90:0/44614321 conn(0x7f7ddc0fc3b0 msgr2=0x7f7ddc105320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:02.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 shutdown_connections 2024-04-06T14:05:02.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:02.367+0000 7f7de12bb700 1 -- 172.21.15.90:0/44614321 wait complete. 2024-04-06T14:05:02.748 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph orch host ls' 2024-04-06T14:05:02.974 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:02 smithi090 ceph-mon[43420]: pgmap v250: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:05:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:02 smithi090 ceph-mon[43420]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:02 smithi134 ceph-mon[47549]: pgmap v250: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:05:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:02 smithi134 ceph-mon[47549]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:02 smithi105 ceph-mon[46969]: pgmap v250: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.9 KiB/s rd, 5 op/s 2024-04-06T14:05:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:02 smithi105 ceph-mon[46969]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:04.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 -- 172.21.15.90:0/2732377223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 msgr2=0x7fd31c100130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:04.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 --2- 172.21.15.90:0/2732377223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c100130 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fd3040099c0 tx=0x7fd304009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 -- 172.21.15.90:0/2732377223 shutdown_connections 2024-04-06T14:05:04.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 --2- 172.21.15.90:0/2732377223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 0x7fd31c108160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 --2- 172.21.15.90:0/2732377223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c100130 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 --2- 172.21.15.90:0/2732377223 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 0x7fd31c101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 -- 172.21.15.90:0/2732377223 >> 172.21.15.90:0/2732377223 conn(0x7fd31c0fb2b0 msgr2=0x7fd31c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 -- 172.21.15.90:0/2732377223 shutdown_connections 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.087+0000 7fd32262e700 1 -- 172.21.15.90:0/2732377223 wait complete. 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 Processor -- start 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 -- start start 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 0x7fd31c197250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 0x7fd31c19e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd31c19e7f0 con 0x7fd31c0ffd10 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd31c19e930 con 0x7fd31c1018b0 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd32262e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd31c19eaf0 con 0x7fd31c100f10 2024-04-06T14:05:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd31bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd31bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36296/0 (socket says 172.21.15.90:36296) 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.088+0000 7fd31bfff700 1 -- 172.21.15.90:0/3600190176 learned_addr learned my addr 172.21.15.90:0/3600190176 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd320bcb700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 0x7fd31c19e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31b7fe700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 0x7fd31c197250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 -- 172.21.15.90:0/3600190176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 msgr2=0x7fd31c197250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 0x7fd31c197250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 -- 172.21.15.90:0/3600190176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 msgr2=0x7fd31c19e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 0x7fd31c19e220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 -- 172.21.15.90:0/3600190176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3040095f0 con 0x7fd31c0ffd10 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd31bfff700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fd304009f50 tx=0x7fd3040057f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd30401d070 con 0x7fd31c0ffd10 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd31c19ed70 con 0x7fd31c0ffd10 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd304004200 con 0x7fd31c0ffd10 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd304022e40 con 0x7fd31c0ffd10 2024-04-06T14:05:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.089+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd31c19f1c0 con 0x7fd31c0ffd10 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.090+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd31c1071f0 con 0x7fd31c0ffd10 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.091+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fd304022570 con 0x7fd31c0ffd10 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.092+0000 7fd3197fa700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 0x7fd3080a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.092+0000 7fd31b7fe700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 0x7fd3080a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.092+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7979+0+0 (secure 0 0 0) 0x7fd3040cdc90 con 0x7fd31c0ffd10 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.092+0000 7fd31b7fe700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 0x7fd3080a3670 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd30c005ea0 tx=0x7fd30c005e30 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.096+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd304097500 con 0x7fd31c0ffd10 2024-04-06T14:05:04.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.305+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) v1 -- 0x7fd31c02cd20 con 0x7fd3080a11b0 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.307+0000 7fd3197fa700 1 -- 172.21.15.90:0/3600190176 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+191 (secure 0 0 0) 0x7fd31c02cd20 con 0x7fd3080a11b0 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stdout:HOST ADDR LABELS STATUS 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stdout:smithi090 172.21.15.90 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stdout:smithi105 172.21.15.105 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stdout:smithi134 172.21.15.134 2024-04-06T14:05:04.306 INFO:teuthology.orchestra.run.smithi090.stdout:3 hosts in cluster 2024-04-06T14:05:04.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 msgr2=0x7fd3080a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:04.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 0x7fd3080a3670 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd30c005ea0 tx=0x7fd30c005e30 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 msgr2=0x7fd31c196d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fd304009f50 tx=0x7fd3040057f0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 shutdown_connections 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd3080a11b0 0x7fd3080a3670 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd31c1018b0 0x7fd31c19e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd31c0ffd10 0x7fd31c196d10 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 --2- 172.21.15.90:0/3600190176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd31c100f10 0x7fd31c197250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 >> 172.21.15.90:0/3600190176 conn(0x7fd31c0fb2b0 msgr2=0x7fd31c104330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 shutdown_connections 2024-04-06T14:05:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:04.309+0000 7fd32262e700 1 -- 172.21.15.90:0/3600190176 wait complete. 2024-04-06T14:05:04.484 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]: dispatch 2024-04-06T14:05:04.485 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:04 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]: dispatch 2024-04-06T14:05:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:04 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]: dispatch 2024-04-06T14:05:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:04 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]: dispatch 2024-04-06T14:05:04.690 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph orch device ls' 2024-04-06T14:05:04.912 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: pgmap v251: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-04-06T14:05:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]': finished 2024-04-06T14:05:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: osdmap e82: 12 total, 12 up, 12 in 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:05.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:05 smithi090 ceph-mon[43420]: osdmap e83: 12 total, 12 up, 12 in 2024-04-06T14:05:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: pgmap v251: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: osdmap e82: 12 total, 12 up, 12 in 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:05 smithi105 ceph-mon[46969]: osdmap e83: 12 total, 12 up, 12 in 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: pgmap v251: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.3 KiB/s rd, 6 op/s 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.a", "id": [2, 11]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [8, 4]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1", "id": [4, 9]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 9]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.e", "id": [4, 6]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 8]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='mgr.14152 172.21.15.90:0/924898470' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.20", "id": [5, 9]}]': finished 2024-04-06T14:05:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: osdmap e82: 12 total, 12 up, 12 in 2024-04-06T14:05:05.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:05.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:05 smithi134 ceph-mon[47549]: osdmap e83: 12 total, 12 up, 12 in 2024-04-06T14:05:06.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 -- 172.21.15.90:0/4235688106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c1018b0 msgr2=0x7f0b0c108160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/4235688106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c108160 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f0b0400b0d0 tx=0x7f0b0400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 -- 172.21.15.90:0/4235688106 shutdown_connections 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/4235688106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c108160 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/4235688106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/4235688106 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 0x7f0b0c101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.162+0000 7f0b11dcf700 1 -- 172.21.15.90:0/4235688106 >> 172.21.15.90:0/4235688106 conn(0x7f0b0c0fb2b0 msgr2=0x7f0b0c0fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- 172.21.15.90:0/4235688106 shutdown_connections 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- 172.21.15.90:0/4235688106 wait complete. 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 Processor -- start 2024-04-06T14:05:06.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- start start 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 0x7f0b0c1995c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c1a05f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b0c1a0bc0 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b0c1a0d00 con 0x7f0b0c1018b0 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.163+0000 7f0b11dcf700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b0c1a0e90 con 0x7f0b0c100f10 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:06.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36306/0 (socket says 172.21.15.90:36306) 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 -- 172.21.15.90:0/1678676859 learned_addr learned my addr 172.21.15.90:0/1678676859 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0bfff700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c1a05f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b02ffd700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 0x7f0b0c1995c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 -- 172.21.15.90:0/1678676859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 msgr2=0x7f0b0c1995c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 0x7f0b0c1995c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 -- 172.21.15.90:0/1678676859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c1018b0 msgr2=0x7f0b0c1a05f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c1a05f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 -- 172.21.15.90:0/1678676859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b04009d00 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b0b7fe700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0afc00c9a0 tx=0x7f0afc00ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0afc004220 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b0c1a1170 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0afc007d20 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0afc003c00 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.164+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b0c1a16c0 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.165+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0b0c04ea80 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.166+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f0afc007860 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.167+0000 7f0b097fa700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 0x7f0af80a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:06.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.167+0000 7f0b02ffd700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 0x7f0af80a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:06.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.167+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f0afc0cc5e0 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.167+0000 7f0b02ffd700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 0x7f0af80a36f0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f0af4006fd0 tx=0x7f0af4008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:06.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.171+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0afc095c20 con 0x7f0b0c0ffd10 2024-04-06T14:05:06.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.374+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 --> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) v1 -- 0x7f0b0c0fd490 con 0x7f0af80a1230 2024-04-06T14:05:06.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.375+0000 7f0b097fa700 1 -- 172.21.15.90:0/1678676859 <== mgr.14152 v2:172.21.15.90:6800/1992032630 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+532 (secure 0 0 0) 0x7f0b0c0fd490 con 0x7f0af80a1230 2024-04-06T14:05:06.375 INFO:teuthology.orchestra.run.smithi090.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2024-04-06T14:05:06.375 INFO:teuthology.orchestra.run.smithi090.stdout:smithi090 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400UK400BGN 372G No 2m ago Has a FileSystem, LVM detected 2024-04-06T14:05:06.375 INFO:teuthology.orchestra.run.smithi090.stdout:smithi105 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400TR400BGN 372G No 119s ago Has a FileSystem, LVM detected 2024-04-06T14:05:06.375 INFO:teuthology.orchestra.run.smithi090.stdout:smithi134 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT62420002400BGN 372G No 102s ago Has a FileSystem, LVM detected 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.377+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 msgr2=0x7f0af80a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.377+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 0x7f0af80a36f0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f0af4006fd0 tx=0x7f0af4008040 comp rx=0 tx=0).stop 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.377+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 msgr2=0x7f0b0c199080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.377+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f0afc00c9a0 tx=0x7f0afc00ccb0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 shutdown_connections 2024-04-06T14:05:06.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f0af80a1230 0x7f0af80a36f0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b0c1018b0 0x7f0b0c1a05f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0b0c0ffd10 0x7f0b0c199080 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 --2- 172.21.15.90:0/1678676859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0b0c100f10 0x7f0b0c1995c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 >> 172.21.15.90:0/1678676859 conn(0x7f0b0c0fb2b0 msgr2=0x7f0b0c0fcd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 shutdown_connections 2024-04-06T14:05:06.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:06.378+0000 7f0b11dcf700 1 -- 172.21.15.90:0/1678676859 wait complete. 2024-04-06T14:05:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:06 smithi090 ceph-mon[43420]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-06T14:05:06.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:06 smithi134 ceph-mon[47549]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-06T14:05:06.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:06 smithi105 ceph-mon[46969]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-06T14:05:06.801 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-06T14:05:06.811 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:05:06.811 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph fs dump' 2024-04-06T14:05:07.034 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:07 smithi090 ceph-mon[43420]: pgmap v254: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:07 smithi090 ceph-mon[43420]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:07 smithi134 ceph-mon[47549]: pgmap v254: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:07 smithi134 ceph-mon[47549]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:07.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:07 smithi105 ceph-mon[46969]: pgmap v254: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:07.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:07 smithi105 ceph-mon[46969]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 -- 172.21.15.90:0/4056635923 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38100e20 msgr2=0x7f4c38101240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/4056635923 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38100e20 0x7f4c38101240 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f4c2c009990 tx=0x7f4c2c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 -- 172.21.15.90:0/4056635923 shutdown_connections 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/4056635923 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38102940 0x7f4c38109260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/4056635923 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38100e20 0x7f4c38101240 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/4056635923 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38101f80 0x7f4c38102400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 -- 172.21.15.90:0/4056635923 >> 172.21.15.90:0/4056635923 conn(0x7f4c380fc3c0 msgr2=0x7f4c380fe800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 -- 172.21.15.90:0/4056635923 shutdown_connections 2024-04-06T14:05:08.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 -- 172.21.15.90:0/4056635923 wait complete. 2024-04-06T14:05:08.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.320+0000 7f4c3daf9700 1 Processor -- start 2024-04-06T14:05:08.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 -- start start 2024-04-06T14:05:08.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38100e20 0x7f4c38196de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:08.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 0x7f4c38197320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c3819e8e0 con 0x7f4c38102940 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c3819ea20 con 0x7f4c38101f80 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c3daf9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c3819ebe0 con 0x7f4c38100e20 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36330/0 (socket says 172.21.15.90:36330) 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 -- 172.21.15.90:0/772117085 learned_addr learned my addr 172.21.15.90:0/772117085 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c377fe700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38100e20 0x7f4c38196de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c36ffd700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 0x7f4c38197320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 -- 172.21.15.90:0/772117085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38100e20 msgr2=0x7f4c38196de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38100e20 0x7f4c38196de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 -- 172.21.15.90:0/772117085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 msgr2=0x7f4c38197320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:08.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 0x7f4c38197320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 -- 172.21.15.90:0/772117085 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c2c0095f0 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c36ffd700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 0x7f4c38197320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.321+0000 7f4c37fff700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f4c3000ada0 tx=0x7f4c3000c330 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.322+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3000f070 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.322+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4c3819eec0 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.322+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4c30007b30 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.322+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c30005440 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.322+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c3819f2d0 con 0x7f4c38102940 2024-04-06T14:05:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.323+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4c3804eaf0 con 0x7f4c38102940 2024-04-06T14:05:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.324+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f4c30004ad0 con 0x7f4c38102940 2024-04-06T14:05:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.324+0000 7f4c34ff9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 0x7f4c200a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.324+0000 7f4c377fe700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 0x7f4c200a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.324+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f4c300cc170 con 0x7f4c38102940 2024-04-06T14:05:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.324+0000 7f4c377fe700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 0x7f4c200a3670 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4c2c00b5c0 tx=0x7f4c2c005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:08.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.328+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4c3804eaf0 con 0x7f4c38102940 2024-04-06T14:05:08.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.568+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f4c3804eaf0 con 0x7f4c38102940 2024-04-06T14:05:08.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.569+0000 7f4c34ff9700 1 -- 172.21.15.90:0/772117085 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 23 v23) v1 ==== 76+0+2899 (secure 0 0 0) 0x7f4c3804eaf0 con 0x7f4c38102940 2024-04-06T14:05:08.568 INFO:teuthology.orchestra.run.smithi090.stdout:e23 2024-04-06T14:05:08.568 INFO:teuthology.orchestra.run.smithi090.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:legacy client fscid: 1 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:Filesystem 'cephfs' (1) 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:fs_name cephfs 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:epoch 23 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:created 2024-04-06T14:04:01.430508+0000 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:modified 2024-04-06T14:04:06.509384+0000 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:tableserver 0 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:root 0 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:session_timeout 300 2024-04-06T14:05:08.569 INFO:teuthology.orchestra.run.smithi090.stdout:session_autoclose 300 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:max_file_size 1099511627776 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:max_xattr_size 65536 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:required_client_features {} 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:last_failure 0 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:last_failure_osd_epoch 0 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:max_mds 3 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:in 0,1,2 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:up {0=24515,1=24490,2=24485} 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:failed 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:damaged 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:stopped 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:data_pools [3] 2024-04-06T14:05:08.570 INFO:teuthology.orchestra.run.smithi090.stdout:metadata_pool 2 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:inline_data disabled 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:balancer 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:bal_rank_mask -1 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:standby_count_wanted 0 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.b{0:24515} state up:active seq 3 addr [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.c{0:24469} state up:standby-replay seq 1 addr [v2:172.21.15.134:6836/659065731,v1:172.21.15.134:6837/659065731] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.i{1:24490} state up:active seq 5 addr [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.e{1:24458} state up:standby-replay seq 1 addr [v2:172.21.15.105:6834/3686963284,v1:172.21.15.105:6835/3686963284] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.h{2:24485} state up:active seq 6 addr [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.f{2:24457} state up:standby-replay seq 1 addr [v2:172.21.15.134:6834/2168599536,v1:172.21.15.134:6835/2168599536] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout:Standby daemons: 2024-04-06T14:05:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.j{-1:14565} state up:standby seq 1 addr [v2:172.21.15.90:6834/1262397636,v1:172.21.15.90:6835/1262397636] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.d{-1:14577} state up:standby seq 1 addr [v2:172.21.15.90:6836/886535758,v1:172.21.15.90:6837/886535758] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.g{-1:14631} state up:standby seq 1 addr [v2:172.21.15.90:6838/1327801998,v1:172.21.15.90:6839/1327801998] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.a{-1:14643} state up:standby seq 1 addr [v2:172.21.15.90:6840/761413365,v1:172.21.15.90:6841/761413365] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.k{-1:24434} state up:standby seq 1 addr [v2:172.21.15.105:6832/2124581639,v1:172.21.15.105:6833/2124581639] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.572 INFO:teuthology.orchestra.run.smithi090.stdout:[mds.l{-1:24439} state up:standby seq 1 addr [v2:172.21.15.134:6832/1837725694,v1:172.21.15.134:6833/1837725694] compat {c=[1],r=[1],i=[7ff]}] 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 msgr2=0x7f4c200a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 0x7f4c200a3670 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f4c2c00b5c0 tx=0x7f4c2c005fb0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 msgr2=0x7f4c3819e310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f4c3000ada0 tx=0x7f4c3000c330 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 shutdown_connections 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f4c200a11b0 0x7f4c200a3670 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c38102940 0x7f4c3819e310 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c38100e20 0x7f4c38196de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 --2- 172.21.15.90:0/772117085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c38101f80 0x7f4c38197320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 >> 172.21.15.90:0/772117085 conn(0x7f4c380fc3c0 msgr2=0x7f4c381053d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 shutdown_connections 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:08.571+0000 7f4c3daf9700 1 -- 172.21.15.90:0/772117085 wait complete. 2024-04-06T14:05:08.573 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:05:08.784 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:08 smithi090 ceph-mon[43420]: pgmap v255: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:08.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:08 smithi134 ceph-mon[47549]: pgmap v255: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:08.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:08 smithi105 ceph-mon[46969]: pgmap v255: 97 pgs: 1 peering, 96 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:09.025 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- bash -c 'ceph osd dump' 2024-04-06T14:05:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:09.743 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/772117085' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-06T14:05:09.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/772117085' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-06T14:05:09.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/772117085' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-06T14:05:10.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 -- 172.21.15.90:0/3802245788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97501018b0 msgr2=0x7f9750108160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:10.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 --2- 172.21.15.90:0/3802245788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97501018b0 0x7f9750108160 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f974800b0d0 tx=0x7f974800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 -- 172.21.15.90:0/3802245788 shutdown_connections 2024-04-06T14:05:10.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 --2- 172.21.15.90:0/3802245788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f97501018b0 0x7f9750108160 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 --2- 172.21.15.90:0/3802245788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97500ffd10 0x7f9750100130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 --2- 172.21.15.90:0/3802245788 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9750100f10 0x7f9750101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 -- 172.21.15.90:0/3802245788 >> 172.21.15.90:0/3802245788 conn(0x7f97500fb2b0 msgr2=0x7f97500fd6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 -- 172.21.15.90:0/3802245788 shutdown_connections 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.319+0000 7f97569e4700 1 -- 172.21.15.90:0/3802245788 wait complete. 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 Processor -- start 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 -- start start 2024-04-06T14:05:10.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f97500ffd10 0x7f9750069270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:10.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:10.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 0x7f975019e140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:10.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f975019e8e0 con 0x7f9750100f10 2024-04-06T14:05:10.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f975019ea50 con 0x7f97500ffd10 2024-04-06T14:05:10.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f97569e4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f975019ec10 con 0x7f97501018b0 2024-04-06T14:05:10.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f974f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:10.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f974f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36348/0 (socket says 172.21.15.90:36348) 2024-04-06T14:05:10.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f974f7fe700 1 -- 172.21.15.90:0/3471971779 learned_addr learned my addr 172.21.15.90:0/3471971779 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:10.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.320+0000 7f974ffff700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f97500ffd10 0x7f9750069270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f9754f81700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 0x7f975019e140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 -- 172.21.15.90:0/3471971779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 msgr2=0x7f975019e140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 0x7f975019e140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 -- 172.21.15.90:0/3471971779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f97500ffd10 msgr2=0x7f9750069270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f97500ffd10 0x7f9750069270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 -- 172.21.15.90:0/3471971779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9748009d00 con 0x7f9750100f10 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f9754f81700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 0x7f975019e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974f7fe700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f973800d950 tx=0x7f973800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9738009940 con 0x7f9750100f10 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f975019eef0 con 0x7f9750100f10 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9738010460 con 0x7f9750100f10 2024-04-06T14:05:10.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f973800f620 con 0x7f9750100f10 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.321+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f975019f360 con 0x7f9750100f10 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.322+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9750107120 con 0x7f9750100f10 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.323+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f97380108d0 con 0x7f9750100f10 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.323+0000 7f974d7fa700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 0x7f973c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.324+0000 7f974ffff700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 0x7f973c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.324+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f97380cc8d0 con 0x7f9750100f10 2024-04-06T14:05:10.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.324+0000 7f974ffff700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 0x7f973c0a3670 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9750100d70 tx=0x7f9740009190 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:10.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.328+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9738095f10 con 0x7f9750100f10 2024-04-06T14:05:10.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.522+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump"} v 0) v1 -- 0x7f975004ea80 con 0x7f9750100f10 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.523+0000 7f974d7fa700 1 -- 172.21.15.90:0/3471971779 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump"}]=0 v83) v1 ==== 56+0+5277 (secure 0 0 0) 0x7f9738099530 con 0x7f9750100f10 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stdout:epoch 83 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stdout:fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stdout:created 2024-04-06T13:58:13.321674+0000 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stdout:modified 2024-04-06T14:05:05.185989+0000 2024-04-06T14:05:10.523 INFO:teuthology.orchestra.run.smithi090.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:crush_version 26 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:full_ratio 0.95 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:backfillfull_ratio 0.9 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:nearfull_ratio 0.85 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:require_min_compat_client luminous 2024-04-06T14:05:10.524 INFO:teuthology.orchestra.run.smithi090.stdout:min_compat_client luminous 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:require_osd_release reef 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:stretch_mode_enabled false 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 12.00 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:pool 2 'cephfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 80 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.25 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:pool 3 'cephfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode off last_change 81 flags hashpspool stripe_width 0 pg_num_min 64 target_size_ratio 0.9 application cephfs read_balance_score 1.50 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:max_osd 12 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:osd.0 up in weight 1 up_from 8 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.90:6802/1162173917,v1:172.21.15.90:6803/1162173917] [v2:172.21.15.90:6804/1162173917,v1:172.21.15.90:6805/1162173917] exists,up c3c2803b-07fd-4e23-b926-5e9dcc735ce7 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:osd.1 up in weight 1 up_from 13 up_thru 82 down_at 0 last_clean_interval [0,0) [v2:172.21.15.105:6800/1321301965,v1:172.21.15.105:6801/1321301965] [v2:172.21.15.105:6802/1321301965,v1:172.21.15.105:6803/1321301965] exists,up efe81cdf-a49b-44a5-aece-bef13902a718 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:osd.2 up in weight 1 up_from 18 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6800/1139095317,v1:172.21.15.134:6801/1139095317] [v2:172.21.15.134:6802/1139095317,v1:172.21.15.134:6803/1139095317] exists,up bb0a1674-931b-4a33-80b7-05b28a2b8f86 2024-04-06T14:05:10.525 INFO:teuthology.orchestra.run.smithi090.stdout:osd.3 up in weight 1 up_from 26 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.90:6810/711321384,v1:172.21.15.90:6811/711321384] [v2:172.21.15.90:6812/711321384,v1:172.21.15.90:6813/711321384] exists,up b2f41b24-c336-46cd-af3a-f3cf7d83e3cf 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.4 up in weight 1 up_from 33 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.105:6808/4184734218,v1:172.21.15.105:6809/4184734218] [v2:172.21.15.105:6810/4184734218,v1:172.21.15.105:6811/4184734218] exists,up 6bb3d511-2b64-45cb-845e-57589ef3fb59 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.5 up in weight 1 up_from 38 up_thru 82 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6808/1329717087,v1:172.21.15.134:6809/1329717087] [v2:172.21.15.134:6810/1329717087,v1:172.21.15.134:6811/1329717087] exists,up 989a2709-b8f9-47e7-9385-54bc5d4287ac 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.6 up in weight 1 up_from 45 up_thru 82 down_at 0 last_clean_interval [0,0) [v2:172.21.15.90:6818/1241504412,v1:172.21.15.90:6819/1241504412] [v2:172.21.15.90:6820/1241504412,v1:172.21.15.90:6821/1241504412] exists,up 5fe9a9e4-959e-4fd9-b50d-701606e38f79 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.7 up in weight 1 up_from 49 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.105:6816/4248271964,v1:172.21.15.105:6817/4248271964] [v2:172.21.15.105:6818/4248271964,v1:172.21.15.105:6819/4248271964] exists,up eef257f2-5d81-4b97-9186-03b54813ccea 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.8 up in weight 1 up_from 56 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6816/549202076,v1:172.21.15.134:6817/549202076] [v2:172.21.15.134:6818/549202076,v1:172.21.15.134:6819/549202076] exists,up 98c09ef0-8533-4ab9-b4ba-51aeb43ebc17 2024-04-06T14:05:10.526 INFO:teuthology.orchestra.run.smithi090.stdout:osd.9 up in weight 1 up_from 63 up_thru 82 down_at 0 last_clean_interval [0,0) [v2:172.21.15.90:6826/154728526,v1:172.21.15.90:6827/154728526] [v2:172.21.15.90:6828/154728526,v1:172.21.15.90:6829/154728526] exists,up 599f84b9-90b8-4225-809c-96218058b7c4 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:osd.10 up in weight 1 up_from 68 up_thru 79 down_at 0 last_clean_interval [0,0) [v2:172.21.15.105:6824/394238659,v1:172.21.15.105:6825/394238659] [v2:172.21.15.105:6826/394238659,v1:172.21.15.105:6827/394238659] exists,up 86cce4f5-4741-4f2b-87e5-0af0d083ae8e 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:osd.11 up in weight 1 up_from 76 up_thru 82 down_at 0 last_clean_interval [0,0) [v2:172.21.15.134:6824/1707163828,v1:172.21.15.134:6825/1707163828] [v2:172.21.15.134:6826/1707163828,v1:172.21.15.134:6827/1707163828] exists,up 1942c563-7632-4c97-b0a2-7f079c532b07 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 2.8 [2,11] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 2.a [2,11] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 2.1f [8,4] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 3.1 [4,9] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 3.7 [3,9] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 3.e [4,6] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 3.1d [4,8] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:pg_upmap_items 3.20 [5,9] 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/888041862 expires 2024-04-07T13:59:04.100187+0000 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/1621883497 expires 2024-04-07T13:59:04.100187+0000 2024-04-06T14:05:10.527 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:6801/2246618981 expires 2024-04-07T13:59:04.100187+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:6801/1491440652 expires 2024-04-07T13:58:42.758062+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:6800/1491440652 expires 2024-04-07T13:58:42.758062+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:6800/2246618981 expires 2024-04-07T13:59:04.100187+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/3538889590 expires 2024-04-07T13:58:42.758062+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/2609293956 expires 2024-04-07T13:59:04.100187+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/3768713918 expires 2024-04-07T13:58:42.758062+0000 2024-04-06T14:05:10.528 INFO:teuthology.orchestra.run.smithi090.stdout:blocklist 172.21.15.90:0/3659831004 expires 2024-04-07T13:58:42.758062+0000 2024-04-06T14:05:10.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.525+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 msgr2=0x7f973c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.525+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 0x7f973c0a3670 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9750100d70 tx=0x7f9740009190 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.525+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 msgr2=0x7f975019bb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.525+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f973800d950 tx=0x7f973800dc60 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 shutdown_connections 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f973c0a11b0 0x7f973c0a3670 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f97501018b0 0x7f975019e140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f97500ffd10 0x7f9750069270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 --2- 172.21.15.90:0/3471971779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9750100f10 0x7f975019bb70 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 >> 172.21.15.90:0/3471971779 conn(0x7f97500fb2b0 msgr2=0x7f9750104260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 shutdown_connections 2024-04-06T14:05:10.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:10.526+0000 7f97569e4700 1 -- 172.21.15.90:0/3471971779 wait complete. 2024-04-06T14:05:10.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:10 smithi090 ceph-mon[43420]: pgmap v256: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:10.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:10 smithi134 ceph-mon[47549]: pgmap v256: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:10.906 INFO:teuthology.run_tasks:Running task fs.ready... 2024-04-06T14:05:10.916 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:10.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:10 smithi105 ceph-mon[46969]: pgmap v256: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.1 KiB/s rd, 6 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:11.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 -- 172.21.15.90:0/708276807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b20106950 msgr2=0x7f9b20108dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 --2- 172.21.15.90:0/708276807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b20106950 0x7f9b20108dc0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f9b1800b0a0 tx=0x7f9b1800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 -- 172.21.15.90:0/708276807 shutdown_connections 2024-04-06T14:05:11.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 --2- 172.21.15.90:0/708276807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b20106950 0x7f9b20108dc0 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 --2- 172.21.15.90:0/708276807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b201016f0 0x7f9b20103ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 --2- 172.21.15.90:0/708276807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20106410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.043+0000 7f9b24e55700 1 -- 172.21.15.90:0/708276807 >> 172.21.15.90:0/708276807 conn(0x7f9b200fb100 msgr2=0x7f9b200fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- 172.21.15.90:0/708276807 shutdown_connections 2024-04-06T14:05:11.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- 172.21.15.90:0/708276807 wait complete. 2024-04-06T14:05:11.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 Processor -- start 2024-04-06T14:05:11.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- start start 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20197290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b20106950 0x7f9b2019e270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b2019e840 con 0x7f9b201016f0 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b2019e9b0 con 0x7f9b20106950 2024-04-06T14:05:11.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.044+0000 7f9b24e55700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b2019eb70 con 0x7f9b20104020 2024-04-06T14:05:11.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36370/0 (socket says 172.21.15.90:36370) 2024-04-06T14:05:11.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 -- 172.21.15.90:0/383278065 learned_addr learned my addr 172.21.15.90:0/383278065 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:11.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1ed9d700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b20106950 0x7f9b2019e270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1dd9b700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20197290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 -- 172.21.15.90:0/383278065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 msgr2=0x7f9b20197290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20197290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 -- 172.21.15.90:0/383278065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b20106950 msgr2=0x7f9b2019e270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b20106950 0x7f9b2019e270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 -- 172.21.15.90:0/383278065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b18009d00 con 0x7f9b201016f0 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1dd9b700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20197290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b1e59c700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f9b1000d980 tx=0x7f9b1000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b10004220 con 0x7f9b201016f0 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b10007640 con 0x7f9b201016f0 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b2019ee50 con 0x7f9b201016f0 2024-04-06T14:05:11.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b10005610 con 0x7f9b201016f0 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.045+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b2019f270 con 0x7f9b201016f0 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.046+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b200fcc80 con 0x7f9b201016f0 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.047+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f9b10004750 con 0x7f9b201016f0 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.048+0000 7f9b177fe700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 0x7f9b0c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.048+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f9b100cce00 con 0x7f9b201016f0 2024-04-06T14:05:11.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.048+0000 7f9b1dd9b700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 0x7f9b0c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.048+0000 7f9b1dd9b700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 0x7f9b0c0a3670 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f9b08007900 tx=0x7f9b08008040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.051+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9b10096440 con 0x7f9b201016f0 2024-04-06T14:05:11.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.249+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f9b2010c500 con 0x7f9b201016f0 2024-04-06T14:05:11.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.250+0000 7f9b177fe700 1 -- 172.21.15.90:0/383278065 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+22483 (secure 0 0 0) 0x7f9b10016090 con 0x7f9b201016f0 2024-04-06T14:05:11.250 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:11.250 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":83,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:05.185989+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:11.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.252+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 msgr2=0x7f9b0c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.252+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 0x7f9b0c0a3670 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f9b08007900 tx=0x7f9b08008040 comp rx=0 tx=0).stop 2024-04-06T14:05:11.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.252+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 msgr2=0x7f9b20196d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.252+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f9b1000d980 tx=0x7f9b1000dc90 comp rx=0 tx=0).stop 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 shutdown_connections 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9b0c0a11b0 0x7f9b0c0a3670 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b20106950 0x7f9b2019e270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b201016f0 0x7f9b20196d50 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 --2- 172.21.15.90:0/383278065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b20104020 0x7f9b20197290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 >> 172.21.15.90:0/383278065 conn(0x7f9b200fb100 msgr2=0x7f9b20101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 shutdown_connections 2024-04-06T14:05:11.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.253+0000 7f9b24e55700 1 -- 172.21.15.90:0/383278065 wait complete. 2024-04-06T14:05:11.265 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:05:11.265 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:05:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- 172.21.15.90:0/3999231159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a880ffc90 msgr2=0x7f7a881000b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- 172.21.15.90:0/3999231159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a880ffc90 0x7f7a881000b0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f7a780099c0 tx=0x7f7a78009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- 172.21.15.90:0/3999231159 shutdown_connections 2024-04-06T14:05:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- 172.21.15.90:0/3999231159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a88101820 0x7f7a881080d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- 172.21.15.90:0/3999231159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a880ffc90 0x7f7a881000b0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- 172.21.15.90:0/3999231159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881012e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- 172.21.15.90:0/3999231159 >> 172.21.15.90:0/3999231159 conn(0x7f7a880fb200 msgr2=0x7f7a880fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- 172.21.15.90:0/3999231159 shutdown_connections 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- 172.21.15.90:0/3999231159 wait complete. 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 Processor -- start 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- start start 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a880ffc90 0x7f7a88196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a8819e8d0 con 0x7f7a88101820 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a8819ea10 con 0x7f7a88100e80 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.391+0000 7f7a8f838700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a8819eba0 con 0x7f7a880ffc90 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36380/0 (socket says 172.21.15.90:36380) 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 -- 172.21.15.90:0/1904894649 learned_addr learned my addr 172.21.15.90:0/1904894649 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8d5d4700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a880ffc90 0x7f7a88196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8cdd3700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 -- 172.21.15.90:0/1904894649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a880ffc90 msgr2=0x7f7a88196da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a880ffc90 0x7f7a88196da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 -- 172.21.15.90:0/1904894649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 msgr2=0x7f7a881972e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881972e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 -- 172.21.15.90:0/1904894649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a780095f0 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8cdd3700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8ddd5700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f7a8000b970 tx=0x7f7a8000bd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a8000c9b0 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a80014440 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a8819ee80 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a800075d0 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.392+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a8819f300 con 0x7f7a88101820 2024-04-06T14:05:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.393+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a88067f50 con 0x7f7a88101820 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.394+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f7a800145b0 con 0x7f7a88101820 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.395+0000 7f7a7e7fc700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 0x7f7a740a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.395+0000 7f7a8d5d4700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 0x7f7a740a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.395+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f7a80006e40 con 0x7f7a88101820 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.395+0000 7f7a8d5d4700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 0x7f7a740a36f0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f7a7800b580 tx=0x7f7a78005e80 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.399+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7a80095b90 con 0x7f7a88101820 2024-04-06T14:05:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:11 smithi090 ceph-mon[43420]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-06T14:05:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:11 smithi090 ceph-mon[43420]: Cluster is now healthy 2024-04-06T14:05:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3471971779' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-06T14:05:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:11 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/383278065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:11.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.599+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f7a8819f6a0 con 0x7f7a88101820 2024-04-06T14:05:11.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.599+0000 7f7a7e7fc700 1 -- 172.21.15.90:0/1904894649 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) v1 ==== 93+0+10 (secure 0 0 0) 0x7f7a80023020 con 0x7f7a88101820 2024-04-06T14:05:11.599 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 msgr2=0x7f7a740a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 0x7f7a740a36f0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f7a7800b580 tx=0x7f7a78005e80 comp rx=0 tx=0).stop 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 msgr2=0x7f7a8819e300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f7a8000b970 tx=0x7f7a8000bd30 comp rx=0 tx=0).stop 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 shutdown_connections 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f7a740a1230 0x7f7a740a36f0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a88101820 0x7f7a8819e300 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a880ffc90 0x7f7a88196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 --2- 172.21.15.90:0/1904894649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a88100e80 0x7f7a881972e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 >> 172.21.15.90:0/1904894649 conn(0x7f7a880fb200 msgr2=0x7f7a88069260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 shutdown_connections 2024-04-06T14:05:11.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.601+0000 7f7a8f838700 1 -- 172.21.15.90:0/1904894649 wait complete. 2024-04-06T14:05:11.613 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:05:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 -- 172.21.15.90:0/4003933840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 msgr2=0x7f965c100090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 --2- 172.21.15.90:0/4003933840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c100090 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f964c0099c0 tx=0x7f964c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 -- 172.21.15.90:0/4003933840 shutdown_connections 2024-04-06T14:05:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 --2- 172.21.15.90:0/4003933840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c101800 0x7f965c1080b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 --2- 172.21.15.90:0/4003933840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c100090 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 --2- 172.21.15.90:0/4003933840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c100e60 0x7f965c1012c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 -- 172.21.15.90:0/4003933840 >> 172.21.15.90:0/4003933840 conn(0x7f965c0fb200 msgr2=0x7f965c0fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 -- 172.21.15.90:0/4003933840 shutdown_connections 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.739+0000 7f966254a700 1 -- 172.21.15.90:0/4003933840 wait complete. 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 Processor -- start 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 -- start start 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c100e60 0x7f965c19ba00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 0x7f965c19dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f965c19e790 con 0x7f965c0ffc70 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f965c19e900 con 0x7f965c100e60 2024-04-06T14:05:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f966254a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f965c19eac0 con 0x7f965c101800 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36392/0 (socket says 172.21.15.90:36392) 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 -- 172.21.15.90:0/3526859872 learned_addr learned my addr 172.21.15.90:0/3526859872 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965b7fe700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c100e60 0x7f965c19ba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f9660ae7700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 0x7f965c19dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 -- 172.21.15.90:0/3526859872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 msgr2=0x7f965c19dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 0x7f965c19dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 -- 172.21.15.90:0/3526859872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c100e60 msgr2=0x7f965c19ba00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c100e60 0x7f965c19ba00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f965bfff700 1 -- 172.21.15.90:0/3526859872 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f964c0095f0 con 0x7f965c0ffc70 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.740+0000 7f9660ae7700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 0x7f965c19dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:11.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f965bfff700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f964c003820 tx=0x7f964c004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f964c01d070 con 0x7f965c0ffc70 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f964c0054f0 con 0x7f965c0ffc70 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f964c004a00 con 0x7f965c0ffc70 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f965c19ed40 con 0x7f965c0ffc70 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.741+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f965c19f1a0 con 0x7f965c0ffc70 2024-04-06T14:05:11.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.742+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f965c107110 con 0x7f965c0ffc70 2024-04-06T14:05:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.743+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f964c003b50 con 0x7f965c0ffc70 2024-04-06T14:05:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.743+0000 7f96597fa700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 0x7f96480a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.744+0000 7f965b7fe700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 0x7f96480a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.744+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f964c0cdcc0 con 0x7f965c0ffc70 2024-04-06T14:05:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.744+0000 7f965b7fe700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 0x7f96480a36f0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f965c068ee0 tx=0x7f964400b410 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:11.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.747+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f964c097330 con 0x7f965c0ffc70 2024-04-06T14:05:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:11 smithi134 ceph-mon[47549]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-06T14:05:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:11 smithi134 ceph-mon[47549]: Cluster is now healthy 2024-04-06T14:05:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:11 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3471971779' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-06T14:05:11.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:11 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/383278065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:11.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.941+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f965c10b480 con 0x7f965c0ffc70 2024-04-06T14:05:11.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.942+0000 7f96597fa700 1 -- 172.21.15.90:0/3526859872 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v83) v1 ==== 104+0+11 (secure 0 0 0) 0x7f964c09a950 con 0x7f965c0ffc70 2024-04-06T14:05:11.941 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:05:11.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.943+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 msgr2=0x7f96480a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.943+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 0x7f96480a36f0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f965c068ee0 tx=0x7f964400b410 comp rx=0 tx=0).stop 2024-04-06T14:05:11.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 msgr2=0x7f965c068920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:11.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f964c003820 tx=0x7f964c004730 comp rx=0 tx=0).stop 2024-04-06T14:05:11.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 shutdown_connections 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f96480a1230 0x7f96480a36f0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f965c101800 0x7f965c19dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f965c0ffc70 0x7f965c068920 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 --2- 172.21.15.90:0/3526859872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f965c100e60 0x7f965c19ba00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 >> 172.21.15.90:0/3526859872 conn(0x7f965c0fb200 msgr2=0x7f965c104250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 shutdown_connections 2024-04-06T14:05:11.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:11.944+0000 7f966254a700 1 -- 172.21.15.90:0/3526859872 wait complete. 2024-04-06T14:05:11.955 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:05:11.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:11 smithi105 ceph-mon[46969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-06T14:05:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:11 smithi105 ceph-mon[46969]: Cluster is now healthy 2024-04-06T14:05:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:11 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3471971779' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-06T14:05:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:11 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/383278065' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- 172.21.15.90:0/3180676313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 msgr2=0x7f9244103c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 --2- 172.21.15.90:0/3180676313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244103c40 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f9234009990 tx=0x7f9234009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- 172.21.15.90:0/3180676313 shutdown_connections 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 --2- 172.21.15.90:0/3180676313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244106ab0 0x7f9244108ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 --2- 172.21.15.90:0/3180676313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244103c40 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 --2- 172.21.15.90:0/3180676313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244104180 0x7f9244106570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- 172.21.15.90:0/3180676313 >> 172.21.15.90:0/3180676313 conn(0x7f92440fb200 msgr2=0x7f92440fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- 172.21.15.90:0/3180676313 shutdown_connections 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- 172.21.15.90:0/3180676313 wait complete. 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 Processor -- start 2024-04-06T14:05:12.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.081+0000 7f9248f85700 1 -- start start 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244104180 0x7f9244197200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 0x7f924419e1e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f924419e7b0 con 0x7f9244101850 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f924419e920 con 0x7f9244104180 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f924419eae0 con 0x7f9244106ab0 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36396/0 (socket says 172.21.15.90:36396) 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 -- 172.21.15.90:0/2084813225 learned_addr learned my addr 172.21.15.90:0/2084813225 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9241d9b700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244104180 0x7f9244197200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9242d9d700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 0x7f924419e1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 -- 172.21.15.90:0/2084813225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 msgr2=0x7f924419e1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 0x7f924419e1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 -- 172.21.15.90:0/2084813225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244104180 msgr2=0x7f9244197200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244104180 0x7f9244197200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 -- 172.21.15.90:0/2084813225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f92340095f0 con 0x7f9244101850 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9242d9d700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 0x7f924419e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f924259c700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f9234000c00 tx=0x7f9234004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f923401d070 con 0x7f9244101850 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f92340054f0 con 0x7f9244101850 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f924419ed60 con 0x7f9244101850 2024-04-06T14:05:12.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9234004cd0 con 0x7f9244101850 2024-04-06T14:05:12.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.082+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f924419f120 con 0x7f9244101850 2024-04-06T14:05:12.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.083+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f92440fcd80 con 0x7f9244101850 2024-04-06T14:05:12.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.084+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f923400fc40 con 0x7f9244101850 2024-04-06T14:05:12.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.085+0000 7f92337fe700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 0x7f922c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.085+0000 7f9241d9b700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 0x7f922c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.085+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f92340cde30 con 0x7f9244101850 2024-04-06T14:05:12.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.085+0000 7f9241d9b700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 0x7f922c0a3670 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f9238005950 tx=0x7f923800a300 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.088+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9234097500 con 0x7f9244101850 2024-04-06T14:05:12.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.285+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f924410c660 con 0x7f9244101850 2024-04-06T14:05:12.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.285+0000 7f92337fe700 1 -- 172.21.15.90:0/2084813225 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v83) v1 ==== 100+0+11 (secure 0 0 0) 0x7f923409ab20 con 0x7f9244101850 2024-04-06T14:05:12.285 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:05:12.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 msgr2=0x7f922c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 0x7f922c0a3670 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f9238005950 tx=0x7f923800a300 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 msgr2=0x7f9244196cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f9234000c00 tx=0x7f9234004730 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 shutdown_connections 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f922c0a11b0 0x7f922c0a3670 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9244106ab0 0x7f924419e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9244101850 0x7f9244196cc0 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 --2- 172.21.15.90:0/2084813225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9244104180 0x7f9244197200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 >> 172.21.15.90:0/2084813225 conn(0x7f92440fb200 msgr2=0x7f92441036c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 shutdown_connections 2024-04-06T14:05:12.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.288+0000 7f9248f85700 1 -- 172.21.15.90:0/2084813225 wait complete. 2024-04-06T14:05:12.300 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:05:12.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 -- 172.21.15.90:0/3988142406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c102820 msgr2=0x7fde6c1090d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 --2- 172.21.15.90:0/3988142406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c102820 0x7fde6c1090d0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7fde6400b0d0 tx=0x7fde6400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 -- 172.21.15.90:0/3988142406 shutdown_connections 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 --2- 172.21.15.90:0/3988142406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c102820 0x7fde6c1090d0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 --2- 172.21.15.90:0/3988142406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c100c90 0x7fde6c1010b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 --2- 172.21.15.90:0/3988142406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c101e80 0x7fde6c1022e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 -- 172.21.15.90:0/3988142406 >> 172.21.15.90:0/3988142406 conn(0x7fde6c0fc220 msgr2=0x7fde6c0fe640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 -- 172.21.15.90:0/3988142406 shutdown_connections 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.422+0000 7fde747eb700 1 -- 172.21.15.90:0/3988142406 wait complete. 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 Processor -- start 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 -- start start 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 0x7fde6c074b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c102820 0x7fde6c0736e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde6c073e80 con 0x7fde6c101e80 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde6c073ff0 con 0x7fde6c100c90 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.423+0000 7fde747eb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde6c1a0d10 con 0x7fde6c102820 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36412/0 (socket says 172.21.15.90:36412) 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 -- 172.21.15.90:0/668350997 learned_addr learned my addr 172.21.15.90:0/668350997 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde72d88700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c102820 0x7fde6c0736e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde72587700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 0x7fde6c074b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 -- 172.21.15.90:0/668350997 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c102820 msgr2=0x7fde6c0736e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c102820 0x7fde6c0736e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 -- 172.21.15.90:0/668350997 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 msgr2=0x7fde6c074b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 0x7fde6c074b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 -- 172.21.15.90:0/668350997 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde64009d00 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde72587700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 0x7fde6c074b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde71d86700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fde5c00c920 tx=0x7fde5c00cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde5c007aa0 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fde5c00ce70 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde6c1a0fa0 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fde5c005440 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.424+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde6c1a1420 con 0x7fde6c101e80 2024-04-06T14:05:12.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.425+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde6c108040 con 0x7fde6c101e80 2024-04-06T14:05:12.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.426+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fde5c0055a0 con 0x7fde6c101e80 2024-04-06T14:05:12.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.427+0000 7fde637fe700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 0x7fde580a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.427+0000 7fde72587700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 0x7fde580a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.427+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fde5c0cc930 con 0x7fde6c101e80 2024-04-06T14:05:12.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.427+0000 7fde72587700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 0x7fde580a3670 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fde6c101ce0 tx=0x7fde68009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.430+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fde5c095f70 con 0x7fde6c101e80 2024-04-06T14:05:12.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.675+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fde6c067a20 con 0x7fde6c101e80 2024-04-06T14:05:12.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.676+0000 7fde637fe700 1 -- 172.21.15.90:0/668350997 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fde5c01d020 con 0x7fde6c101e80 2024-04-06T14:05:12.676 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:12.676 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:05:12.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.678+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 msgr2=0x7fde580a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.678+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 0x7fde580a3670 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7fde6c101ce0 tx=0x7fde68009260 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.678+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 msgr2=0x7fde6c0731a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.678+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fde5c00c920 tx=0x7fde5c00cc30 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 shutdown_connections 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fde580a11b0 0x7fde580a3670 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fde6c102820 0x7fde6c0736e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fde6c100c90 0x7fde6c074b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 --2- 172.21.15.90:0/668350997 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fde6c101e80 0x7fde6c0731a0 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 >> 172.21.15.90:0/668350997 conn(0x7fde6c0fc220 msgr2=0x7fde6c105180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 shutdown_connections 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.679+0000 7fde747eb700 1 -- 172.21.15.90:0/668350997 wait complete. 2024-04-06T14:05:12.679 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:05:12.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:12 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1904894649' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.689 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:12 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3526859872' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:12 smithi090 ceph-mon[43420]: pgmap v257: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:12.690 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:12 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2084813225' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.691 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.813+0000 7f113e9a6700 1 -- 172.21.15.90:0/2419474674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1138100d60 msgr2=0x7f11381011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.813+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2419474674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1138100d60 0x7f11381011c0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f11280099c0 tx=0x7f1128009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 -- 172.21.15.90:0/2419474674 shutdown_connections 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2419474674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138101700 0x7f1138107fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2419474674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f11380ffb70 0x7f11380fff90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2419474674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1138100d60 0x7f11381011c0 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 -- 172.21.15.90:0/2419474674 >> 172.21.15.90:0/2419474674 conn(0x7f11380fb100 msgr2=0x7f11380fd520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 -- 172.21.15.90:0/2419474674 shutdown_connections 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 -- 172.21.15.90:0/2419474674 wait complete. 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 Processor -- start 2024-04-06T14:05:12.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.814+0000 7f113e9a6700 1 -- start start 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138100d60 0x7f113819b950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 0x7f113819df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f113819e760 con 0x7f11380ffb70 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f113819e8d0 con 0x7f1138101700 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f113819ea90 con 0x7f1138100d60 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36440/0 (socket says 172.21.15.90:36440) 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 -- 172.21.15.90:0/2285309510 learned_addr learned my addr 172.21.15.90:0/2285309510 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113cf43700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 0x7f113819df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f11377fe700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138100d60 0x7f113819b950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 -- 172.21.15.90:0/2285309510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138100d60 msgr2=0x7f113819b950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138100d60 0x7f113819b950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 -- 172.21.15.90:0/2285309510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 msgr2=0x7f113819df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:12.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 0x7f113819df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 -- 172.21.15.90:0/2285309510 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11280095f0 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113cf43700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 0x7f113819df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f1137fff700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f112000d980 tx=0x7f112000dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f112000de60 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11200044e0 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f113819ed10 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1120003be0 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.815+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f113819f1a0 con 0x7f11380ffb70 2024-04-06T14:05:12.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.816+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1138106ff0 con 0x7f11380ffb70 2024-04-06T14:05:12.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.817+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f1120009bb0 con 0x7f11380ffb70 2024-04-06T14:05:12.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.818+0000 7f11357fa700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 0x7f11240a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:12.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.818+0000 7f11377fe700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 0x7f11240a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:12.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.818+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f11200ccb70 con 0x7f11380ffb70 2024-04-06T14:05:12.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.818+0000 7f11377fe700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 0x7f11240a3670 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f1128000c00 tx=0x7f1128005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:12.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:12.821+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f11200961b0 con 0x7f11380ffb70 2024-04-06T14:05:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:12 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1904894649' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:12 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3526859872' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:12 smithi134 ceph-mon[47549]: pgmap v257: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:12.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:12 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2084813225' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:12 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1904894649' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:12 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3526859872' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:12 smithi105 ceph-mon[46969]: pgmap v257: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 2.5 KiB/s rd, 4 op/s; 72 B/s, 0 objects/s recovering 2024-04-06T14:05:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:12 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2084813225' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:13.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.019+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f113810b400 con 0x7f11380ffb70 2024-04-06T14:05:13.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.020+0000 7f11357fa700 1 -- 172.21.15.90:0/2285309510 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+22483 (secure 0 0 0) 0x7f1120009940 con 0x7f11380ffb70 2024-04-06T14:05:13.020 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:13.020 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":83,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:05.185989+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 msgr2=0x7f11240a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 0x7f11240a3670 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f1128000c00 tx=0x7f1128005fb0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 msgr2=0x7f1138068940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f112000d980 tx=0x7f112000dd40 comp rx=0 tx=0).stop 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 shutdown_connections 2024-04-06T14:05:13.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f11240a11b0 0x7f11240a3670 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1138101700 0x7f113819df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f11380ffb70 0x7f1138068940 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 --2- 172.21.15.90:0/2285309510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1138100d60 0x7f113819b950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 >> 172.21.15.90:0/2285309510 conn(0x7f11380fb100 msgr2=0x7f1138104130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 shutdown_connections 2024-04-06T14:05:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.022+0000 7f113e9a6700 1 -- 172.21.15.90:0/2285309510 wait complete. 2024-04-06T14:05:13.038 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:05:13.039 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:05:13.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- 172.21.15.90:0/3848613038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 msgr2=0x7f8988103be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 --2- 172.21.15.90:0/3848613038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988103be0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f8978009990 tx=0x7f8978009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- 172.21.15.90:0/3848613038 shutdown_connections 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 --2- 172.21.15.90:0/3848613038 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 0x7f8988108ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 --2- 172.21.15.90:0/3848613038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988103be0 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 --2- 172.21.15.90:0/3848613038 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 0x7f8988106510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- 172.21.15.90:0/3848613038 >> 172.21.15.90:0/3848613038 conn(0x7f89880fb200 msgr2=0x7f89880fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- 172.21.15.90:0/3848613038 shutdown_connections 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- 172.21.15.90:0/3848613038 wait complete. 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 Processor -- start 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.164+0000 7f898f416700 1 -- start start 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 0x7f8988197160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 0x7f898819e180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f898819e750 con 0x7f89881017f0 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f898819e890 con 0x7f8988106a50 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f898819ea20 con 0x7f8988104120 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36458/0 (socket says 172.21.15.90:36458) 2024-04-06T14:05:13.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 -- 172.21.15.90:0/1568756123 learned_addr learned my addr 172.21.15.90:0/1568756123 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898c9b1700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 0x7f8988197160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d9b3700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 0x7f898819e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 -- 172.21.15.90:0/1568756123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 msgr2=0x7f8988197160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 0x7f8988197160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 -- 172.21.15.90:0/1568756123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 msgr2=0x7f898819e180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 0x7f898819e180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 -- 172.21.15.90:0/1568756123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89780095f0 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898d1b2700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f89780098e0 tx=0x7f8978005720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f897801d070 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8978022470 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f898819eca0 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8978017650 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.165+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f898819f120 con 0x7f89881017f0 2024-04-06T14:05:13.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.166+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f89880fcd80 con 0x7f89881017f0 2024-04-06T14:05:13.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.167+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f89780225e0 con 0x7f89881017f0 2024-04-06T14:05:13.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.168+0000 7f897e7fc700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 0x7f89740a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.168+0000 7f898c9b1700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 0x7f89740a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.168+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f89780cde90 con 0x7f89881017f0 2024-04-06T14:05:13.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.168+0000 7f898c9b1700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 0x7f89740a36f0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f898400ac50 tx=0x7f898400a380 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.172+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8978097590 con 0x7f89881017f0 2024-04-06T14:05:13.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.372+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f898810c550 con 0x7f89881017f0 2024-04-06T14:05:13.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.372+0000 7f897e7fc700 1 -- 172.21.15.90:0/1568756123 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v83) v1 ==== 93+0+10 (secure 0 0 0) 0x7f89780270a0 con 0x7f89881017f0 2024-04-06T14:05:13.372 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 msgr2=0x7f89740a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 0x7f89740a36f0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f898400ac50 tx=0x7f898400a380 comp rx=0 tx=0).stop 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 msgr2=0x7f8988196c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f89780098e0 tx=0x7f8978005720 comp rx=0 tx=0).stop 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 shutdown_connections 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f89740a1230 0x7f89740a36f0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8988106a50 0x7f898819e180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89881017f0 0x7f8988196c20 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 --2- 172.21.15.90:0/1568756123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8988104120 0x7f8988197160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 >> 172.21.15.90:0/1568756123 conn(0x7f89880fb200 msgr2=0x7f8988102390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 shutdown_connections 2024-04-06T14:05:13.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.374+0000 7f898f416700 1 -- 172.21.15.90:0/1568756123 wait complete. 2024-04-06T14:05:13.386 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 -- 172.21.15.90:0/1660963862 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c106b10 msgr2=0x7ff18c108f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 --2- 172.21.15.90:0/1660963862 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c106b10 0x7ff18c108f00 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7ff18400b0a0 tx=0x7ff18400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 -- 172.21.15.90:0/1660963862 shutdown_connections 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 --2- 172.21.15.90:0/1660963862 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c106b10 0x7ff18c108f00 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 --2- 172.21.15.90:0/1660963862 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 0x7ff18c103ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 --2- 172.21.15.90:0/1660963862 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c1041e0 0x7ff18c1065d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.515+0000 7ff191852700 1 -- 172.21.15.90:0/1660963862 >> 172.21.15.90:0/1660963862 conn(0x7ff18c0fb200 msgr2=0x7ff18c0fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- 172.21.15.90:0/1660963862 shutdown_connections 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- 172.21.15.90:0/1660963862 wait complete. 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 Processor -- start 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- start start 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 0x7ff18c198fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 0x7ff18c1a0530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff18c1a0b00 con 0x7ff18c1041e0 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff18c1a0c40 con 0x7ff18c106b10 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff191852700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff18c1a0dd0 con 0x7ff18c1018b0 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff18a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff18a7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40056/0 (socket says 172.21.15.90:40056) 2024-04-06T14:05:13.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.516+0000 7ff18a7fc700 1 -- 172.21.15.90:0/3317354324 learned_addr learned my addr 172.21.15.90:0/3317354324 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18affd700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 0x7ff18c198fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18b7fe700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 0x7ff18c1a0530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 -- 172.21.15.90:0/3317354324 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 msgr2=0x7ff18c198fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 0x7ff18c198fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 -- 172.21.15.90:0/3317354324 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 msgr2=0x7ff18c1a0530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 0x7ff18c1a0530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 -- 172.21.15.90:0/3317354324 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff184009d00 con 0x7ff18c1041e0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18b7fe700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 0x7ff18c1a0530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff18a7fc700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7ff18000d950 tx=0x7ff18000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff180009940 con 0x7ff18c1041e0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff180010460 con 0x7ff18c1041e0 2024-04-06T14:05:13.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff18000f620 con 0x7ff18c1041e0 2024-04-06T14:05:13.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff18c1a10b0 con 0x7ff18c1041e0 2024-04-06T14:05:13.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.517+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff18c1a1570 con 0x7ff18c1041e0 2024-04-06T14:05:13.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.518+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff18c108690 con 0x7ff18c1041e0 2024-04-06T14:05:13.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.519+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7ff1800108d0 con 0x7ff18c1041e0 2024-04-06T14:05:13.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.520+0000 7ff190850700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 0x7ff1740a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.520+0000 7ff18affd700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 0x7ff1740a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.520+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7ff1800cc960 con 0x7ff18c1041e0 2024-04-06T14:05:13.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.520+0000 7ff18affd700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 0x7ff1740a36f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ff17c009c60 tx=0x7ff17c009260 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.524+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff180095f40 con 0x7ff18c1041e0 2024-04-06T14:05:13.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.722+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7ff18c10c570 con 0x7ff18c1041e0 2024-04-06T14:05:13.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.723+0000 7ff190850700 1 -- 172.21.15.90:0/3317354324 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v83) v1 ==== 104+0+11 (secure 0 0 0) 0x7ff180016020 con 0x7ff18c1041e0 2024-04-06T14:05:13.723 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 msgr2=0x7ff1740a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 0x7ff1740a36f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ff17c009c60 tx=0x7ff17c009260 comp rx=0 tx=0).stop 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 msgr2=0x7ff18c199510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7ff18000d950 tx=0x7ff18000dc60 comp rx=0 tx=0).stop 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 shutdown_connections 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7ff1740a1230 0x7ff1740a36f0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff18c106b10 0x7ff18c1a0530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff18c1018b0 0x7ff18c198fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 --2- 172.21.15.90:0/3317354324 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff18c1041e0 0x7ff18c199510 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 >> 172.21.15.90:0/3317354324 conn(0x7ff18c0fb200 msgr2=0x7ff18c1010c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 shutdown_connections 2024-04-06T14:05:13.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.725+0000 7ff191852700 1 -- 172.21.15.90:0/3317354324 wait complete. 2024-04-06T14:05:13.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:13 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/668350997' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:13 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2285309510' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.736 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:13 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1568756123' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:13.737 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:05:13.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 -- 172.21.15.90:0/1126265507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac8102000 msgr2=0x7f3ac8102460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 --2- 172.21.15.90:0/1126265507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac8102000 0x7f3ac8102460 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f3abc009990 tx=0x7f3abc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 -- 172.21.15.90:0/1126265507 shutdown_connections 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 --2- 172.21.15.90:0/1126265507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac81029a0 0x7f3ac8109250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 --2- 172.21.15.90:0/1126265507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac80ff030 0x7f3ac80ff450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 --2- 172.21.15.90:0/1126265507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac8102000 0x7f3ac8102460 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 -- 172.21.15.90:0/1126265507 >> 172.21.15.90:0/1126265507 conn(0x7f3ac80fa960 msgr2=0x7f3ac80fcda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 -- 172.21.15.90:0/1126265507 shutdown_connections 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.862+0000 7f3acdac3700 1 -- 172.21.15.90:0/1126265507 wait complete. 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 Processor -- start 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 -- start start 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac8102000 0x7f3ac8199460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac81029a0 0x7f3ac81a0480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ac81a0a50 con 0x7f3ac80ff030 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ac81a0b90 con 0x7f3ac81029a0 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3acdac3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ac81a0d20 con 0x7f3ac8102000 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3ac77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3ac77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40070/0 (socket says 172.21.15.90:40070) 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3ac77fe700 1 -- 172.21.15.90:0/3683821243 learned_addr learned my addr 172.21.15.90:0/3683821243 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.863+0000 7f3ac7fff700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac81029a0 0x7f3ac81a0480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac6ffd700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac8102000 0x7f3ac8199460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 -- 172.21.15.90:0/3683821243 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac8102000 msgr2=0x7f3ac8199460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac8102000 0x7f3ac8199460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 -- 172.21.15.90:0/3683821243 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac81029a0 msgr2=0x7f3ac81a0480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac81029a0 0x7f3ac81a0480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 -- 172.21.15.90:0/3683821243 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3abc0095f0 con 0x7f3ac80ff030 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac77fe700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f3ab800b500 tx=0x7f3ab800b810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ab8011830 con 0x7f3ac80ff030 2024-04-06T14:05:13.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3ab8011e50 con 0x7f3ac80ff030 2024-04-06T14:05:13.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3ab800f2e0 con 0x7f3ac80ff030 2024-04-06T14:05:13.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ac81a0fa0 con 0x7f3ac80ff030 2024-04-06T14:05:13.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.864+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ac81a1420 con 0x7f3ac80ff030 2024-04-06T14:05:13.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.865+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ac8067270 con 0x7f3ac80ff030 2024-04-06T14:05:13.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.866+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f3ab8010580 con 0x7f3ac80ff030 2024-04-06T14:05:13.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.867+0000 7f3ac4ff9700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 0x7f3ab00a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:13.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.867+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f3ab80cc7f0 con 0x7f3ac80ff030 2024-04-06T14:05:13.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.867+0000 7f3ac6ffd700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 0x7f3ab00a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:13.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.867+0000 7f3ac6ffd700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 0x7f3ab00a36f0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f3abc0037f0 tx=0x7f3abc005fb0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:13.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:13.870+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3ab8095f40 con 0x7f3ac80ff030 2024-04-06T14:05:13.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:13 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/668350997' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:13 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2285309510' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:13 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1568756123' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:13.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:13 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/668350997' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:13 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2285309510' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:13.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:13 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1568756123' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.068+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f3ac810c7a0 con 0x7f3ac80ff030 2024-04-06T14:05:14.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.068+0000 7f3ac4ff9700 1 -- 172.21.15.90:0/3683821243 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v83) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3ac810c7a0 con 0x7f3ac80ff030 2024-04-06T14:05:14.068 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 msgr2=0x7f3ab00a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 0x7f3ab00a36f0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f3abc0037f0 tx=0x7f3abc005fb0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 msgr2=0x7f3ac8198f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f3ab800b500 tx=0x7f3ab800b810 comp rx=0 tx=0).stop 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 shutdown_connections 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f3ab00a1230 0x7f3ab00a36f0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3ac81029a0 0x7f3ac81a0480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3ac80ff030 0x7f3ac8198f20 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 --2- 172.21.15.90:0/3683821243 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3ac8102000 0x7f3ac8199460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.070+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 >> 172.21.15.90:0/3683821243 conn(0x7f3ac80fa960 msgr2=0x7f3ac80ffdf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.071+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 shutdown_connections 2024-04-06T14:05:14.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.071+0000 7f3acdac3700 1 -- 172.21.15.90:0/3683821243 wait complete. 2024-04-06T14:05:14.082 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 -- 172.21.15.90:0/64274139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb318100d90 msgr2=0x7fb3181011f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 --2- 172.21.15.90:0/64274139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb318100d90 0x7fb3181011f0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7fb3140099c0 tx=0x7fb314009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 -- 172.21.15.90:0/64274139 shutdown_connections 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 --2- 172.21.15.90:0/64274139 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb318107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 --2- 172.21.15.90:0/64274139 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3180ffba0 0x7fb3180fffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 --2- 172.21.15.90:0/64274139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb318100d90 0x7fb3181011f0 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 -- 172.21.15.90:0/64274139 >> 172.21.15.90:0/64274139 conn(0x7fb3180fb110 msgr2=0x7fb3180fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 -- 172.21.15.90:0/64274139 shutdown_connections 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.204+0000 7fb3207a8700 1 -- 172.21.15.90:0/64274139 wait complete. 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 Processor -- start 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 -- start start 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb318100d90 0x7fb31819b9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb31819dfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb31819e750 con 0x7fb3180ffba0 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb31819e8c0 con 0x7fb318100d90 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb3207a8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb31819ea80 con 0x7fb318101730 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40092/0 (socket says 172.21.15.90:40092) 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 -- 172.21.15.90:0/2092231952 learned_addr learned my addr 172.21.15.90:0/2092231952 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31dd43700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb318100d90 0x7fb31819b9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 -- 172.21.15.90:0/2092231952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 msgr2=0x7fb31819dfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31ed45700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb31819dfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb31819dfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 -- 172.21.15.90:0/2092231952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb318100d90 msgr2=0x7fb31819b9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb318100d90 0x7fb31819b9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31e544700 1 -- 172.21.15.90:0/2092231952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3140095f0 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.205+0000 7fb31ed45700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb31819dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb31e544700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fb30c00b500 tx=0x7fb30c00b810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb30c011830 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb31819ed60 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb30c011e50 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb30c00f2e0 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.206+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb31819f190 con 0x7fb3180ffba0 2024-04-06T14:05:14.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.207+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb318107020 con 0x7fb3180ffba0 2024-04-06T14:05:14.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.209+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fb30c010580 con 0x7fb3180ffba0 2024-04-06T14:05:14.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.209+0000 7fb30b7fe700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 0x7fb3040a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.209+0000 7fb31dd43700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 0x7fb3040a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.209+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fb30c0cc940 con 0x7fb3180ffba0 2024-04-06T14:05:14.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.210+0000 7fb31dd43700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 0x7fb3040a3670 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fb314009990 tx=0x7fb314005bc0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.213+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb30c096090 con 0x7fb3180ffba0 2024-04-06T14:05:14.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.452+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb318067190 con 0x7fb3180ffba0 2024-04-06T14:05:14.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.452+0000 7fb30b7fe700 1 -- 172.21.15.90:0/2092231952 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fb30c015090 con 0x7fb3180ffba0 2024-04-06T14:05:14.452 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:14.452 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:04:06.509384+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:05:14.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 msgr2=0x7fb3040a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 0x7fb3040a3670 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fb314009990 tx=0x7fb314005bc0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 msgr2=0x7fb318069b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7fb30c00b500 tx=0x7fb30c00b810 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 shutdown_connections 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fb3040a11b0 0x7fb3040a3670 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb318101730 0x7fb31819dfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3180ffba0 0x7fb318069b20 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 --2- 172.21.15.90:0/2092231952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb318100d90 0x7fb31819b9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.454+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 >> 172.21.15.90:0/2092231952 conn(0x7fb3180fb110 msgr2=0x7fb318104160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.455+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 shutdown_connections 2024-04-06T14:05:14.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.455+0000 7fb3207a8700 1 -- 172.21.15.90:0/2092231952 wait complete. 2024-04-06T14:05:14.456 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 23 2024-04-06T14:05:14.467 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:14.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 -- 172.21.15.90:0/3903217152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc102820 msgr2=0x7fc5fc1090d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 --2- 172.21.15.90:0/3903217152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc102820 0x7fc5fc1090d0 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fc5f400b0d0 tx=0x7fc5f400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 -- 172.21.15.90:0/3903217152 shutdown_connections 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 --2- 172.21.15.90:0/3903217152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc102820 0x7fc5fc1090d0 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 --2- 172.21.15.90:0/3903217152 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5fc100c90 0x7fc5fc1010b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 --2- 172.21.15.90:0/3903217152 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5fc101e80 0x7fc5fc1022e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 -- 172.21.15.90:0/3903217152 >> 172.21.15.90:0/3903217152 conn(0x7fc5fc0fc220 msgr2=0x7fc5fc0fe640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 -- 172.21.15.90:0/3903217152 shutdown_connections 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 -- 172.21.15.90:0/3903217152 wait complete. 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.591+0000 7fc601fac700 1 Processor -- start 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 -- start start 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5fc101e80 0x7fc5fc197210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5fc102820 0x7fc5fc19e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5fc19e7f0 con 0x7fc5fc100c90 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5fc19e960 con 0x7fc5fc102820 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc601fac700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5fc19eb20 con 0x7fc5fc101e80 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40118/0 (socket says 172.21.15.90:40118) 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 -- 172.21.15.90:0/859122633 learned_addr learned my addr 172.21.15.90:0/859122633 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 -- 172.21.15.90:0/859122633 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5fc101e80 msgr2=0x7fc5fc197210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5fc101e80 0x7fc5fc197210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 -- 172.21.15.90:0/859122633 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5fc102820 msgr2=0x7fc5fc19e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5fc102820 0x7fc5fc19e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 -- 172.21.15.90:0/859122633 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5f4009d00 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5fb7fe700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fc5ec00c970 tx=0x7fc5ec00cd30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.592+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5ec004220 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.593+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc5ec007d60 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.593+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5fc19ee00 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.593+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5ec003d00 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.593+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5fc19f280 con 0x7fc5fc100c90 2024-04-06T14:05:14.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.593+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc5fc1080e0 con 0x7fc5fc100c90 2024-04-06T14:05:14.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.594+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fc5ec004380 con 0x7fc5fc100c90 2024-04-06T14:05:14.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.595+0000 7fc5f8ff9700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 0x7fc5e80a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.595+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fc5ec0cca30 con 0x7fc5fc100c90 2024-04-06T14:05:14.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.595+0000 7fc5faffd700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 0x7fc5e80a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.595+0000 7fc5faffd700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 0x7fc5e80a3670 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7fc5e4005950 tx=0x7fc5e40058e0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.599+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc5ec096070 con 0x7fc5fc100c90 2024-04-06T14:05:14.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:14 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3317354324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:14 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3683821243' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:14 smithi090 ceph-mon[43420]: pgmap v258: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s; 58 B/s, 0 objects/s recovering 2024-04-06T14:05:14.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:14 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2092231952' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:14.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.801+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc5fc10c530 con 0x7fc5fc100c90 2024-04-06T14:05:14.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.802+0000 7fc5f8ff9700 1 -- 172.21.15.90:0/859122633 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v83) v1 ==== 74+0+22483 (secure 0 0 0) 0x7fc5ec078e60 con 0x7fc5fc100c90 2024-04-06T14:05:14.802 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:14.802 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":83,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:05.185989+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:14.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 msgr2=0x7fc5e80a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 0x7fc5e80a3670 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7fc5e4005950 tx=0x7fc5e40058e0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 msgr2=0x7fc5fc196cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fc5ec00c970 tx=0x7fc5ec00cd30 comp rx=0 tx=0).stop 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 shutdown_connections 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fc5e80a11b0 0x7fc5e80a3670 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5fc102820 0x7fc5fc19e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5fc100c90 0x7fc5fc196cd0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 --2- 172.21.15.90:0/859122633 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5fc101e80 0x7fc5fc197210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 >> 172.21.15.90:0/859122633 conn(0x7fc5fc0fc220 msgr2=0x7fc5fc105220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 shutdown_connections 2024-04-06T14:05:14.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.805+0000 7fc601fac700 1 -- 172.21.15.90:0/859122633 wait complete. 2024-04-06T14:05:14.817 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 23, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False, 'balance_automate': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2024-04-06T14:04:01.430508+0000', 'modified': '2024-04-06T14:04:06.509384+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 3, 'in': [0, 1, 2], 'up': {'mds_0': 24515, 'mds_1': 24490, 'mds_2': 24485}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24457': {'gid': 24457, 'name': 'f', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.134:6835/2168599536', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6834', 'nonce': 2168599536}, {'type': 'v1', 'addr': '172.21.15.134:6835', 'nonce': 2168599536}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24458': {'gid': 24458, 'name': 'e', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.105:6835/3686963284', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6834', 'nonce': 3686963284}, {'type': 'v1', 'addr': '172.21.15.105:6835', 'nonce': 3686963284}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24469': {'gid': 24469, 'name': 'c', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.134:6837/659065731', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6836', 'nonce': 659065731}, {'type': 'v1', 'addr': '172.21.15.134:6837', 'nonce': 659065731}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24485': {'gid': 24485, 'name': 'h', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 6, 'addr': '172.21.15.105:6837/2851238848', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6836', 'nonce': 2851238848}, {'type': 'v1', 'addr': '172.21.15.105:6837', 'nonce': 2851238848}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24490': {'gid': 24490, 'name': 'i', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.134:6839/1913582469', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.134:6838', 'nonce': 1913582469}, {'type': 'v1', 'addr': '172.21.15.134:6839', 'nonce': 1913582469}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24515': {'gid': 24515, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.105:6839/2253165057', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6838', 'nonce': 2253165057}, {'type': 'v1', 'addr': '172.21.15.105:6839', 'nonce': 2253165057}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0} 2024-04-06T14:05:14.817 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 3/3 2024-04-06T14:05:14.818 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2024-04-06T14:05:14.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:14 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3317354324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:14 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3683821243' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:14 smithi134 ceph-mon[47549]: pgmap v258: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s; 58 B/s, 0 objects/s recovering 2024-04-06T14:05:14.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:14 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2092231952' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 -- 172.21.15.90:0/430715081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 msgr2=0x7f94e81012c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 --2- 172.21.15.90:0/430715081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81012c0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f94d00099c0 tx=0x7f94d0009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 -- 172.21.15.90:0/430715081 shutdown_connections 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 --2- 172.21.15.90:0/430715081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 0x7f94e81080b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 --2- 172.21.15.90:0/430715081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 0x7f94e8100090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 --2- 172.21.15.90:0/430715081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81012c0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 -- 172.21.15.90:0/430715081 >> 172.21.15.90:0/430715081 conn(0x7f94e80fb200 msgr2=0x7f94e80fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 -- 172.21.15.90:0/430715081 shutdown_connections 2024-04-06T14:05:14.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.940+0000 7f94eca96700 1 -- 172.21.15.90:0/430715081 wait complete. 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 Processor -- start 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 -- start start 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 0x7f94e8196d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 0x7f94e819e280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f94e819e850 con 0x7f94e8100e60 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f94e819e9c0 con 0x7f94e8101800 2024-04-06T14:05:14.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94eca96700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f94e819eb80 con 0x7f94e80ffc70 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40140/0 (socket says 172.21.15.90:40140) 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 -- 172.21.15.90:0/42161348 learned_addr learned my addr 172.21.15.90:0/42161348 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 0x7f94e819e280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e659c700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 0x7f94e8196d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 -- 172.21.15.90:0/42161348 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 msgr2=0x7f94e8196d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 0x7f94e8196d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 -- 172.21.15.90:0/42161348 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 msgr2=0x7f94e819e280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 0x7f94e819e280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.941+0000 7f94e5d9b700 1 -- 172.21.15.90:0/42161348 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f94d00095f0 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f94d0009f50 tx=0x7f94d0003970 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94df7fe700 1 -- 172.21.15.90:0/42161348 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d001d070 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94df7fe700 1 -- 172.21.15.90:0/42161348 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f94d0022470 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f94e819ee00 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94df7fe700 1 -- 172.21.15.90:0/42161348 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d000f6c0 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.942+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f94e819f1d0 con 0x7f94e8100e60 2024-04-06T14:05:14.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.944+0000 7f94df7fe700 1 -- 172.21.15.90:0/42161348 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f94d00225e0 con 0x7f94e8100e60 2024-04-06T14:05:14.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.944+0000 7f94df7fe700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 0x7f94d40a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.944+0000 7f94df7fe700 1 -- 172.21.15.90:0/42161348 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f94d00cded0 con 0x7f94e8100e60 2024-04-06T14:05:14.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.944+0000 7f94e659c700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 0x7f94d40a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.945+0000 7f94e659c700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 0x7f94d40a3670 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f94d8005fd0 tx=0x7f94d8005dd0 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.945+0000 7f94f8c4ab80 1 Processor -- start 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.945+0000 7f94f8c4ab80 1 -- start start 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb224970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb2253b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb17b940 con 0x5593fb223a00 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb179300 con 0x5593fb224360 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb174bf0 con 0x5593fb224f50 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40150/0 (socket says 172.21.15.90:40150) 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 -- 172.21.15.90:0/102263644 learned_addr learned my addr 172.21.15.90:0/102263644 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb224970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 -- 172.21.15.90:0/102263644 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 msgr2=0x5593fb2253b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb2253b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 -- 172.21.15.90:0/102263644 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 msgr2=0x5593fb224970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb224970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 -- 172.21.15.90:0/102263644 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5593faff6360 con 0x5593fb223a00 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94e659c700 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f94d800a2e0 tx=0x7f94d8011000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e334f8bfc8a0319 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94dcff9700 1 -- 172.21.15.90:0/102263644 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d8016040 con 0x5593fb223a00 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94dcff9700 1 -- 172.21.15.90:0/102263644 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f94d8003d40 con 0x5593fb223a00 2024-04-06T14:05:14.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.946+0000 7f94dcff9700 1 -- 172.21.15.90:0/102263644 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d801eda0 con 0x5593fb223a00 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/102263644 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 msgr2=0x5593fb223e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7f94d800a2e0 tx=0x7f94d8011000 comp rx=0 tx=0).stop 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/102263644 shutdown_connections 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb2253b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb223e20 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/102263644 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb224970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/102263644 >> 172.21.15.90:0/102263644 conn(0x5593fb182650 msgr2=0x5593fb21e560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/102263644 shutdown_connections 2024-04-06T14:05:14.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/102263644 wait complete. 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.947+0000 7f94f8c4ab80 1 -- start start 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 Processor -- start 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb22db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb234ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb110d30 con 0x5593fb223a00 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb1109d0 con 0x5593fb224360 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593faff8650 con 0x5593fb224f50 2024-04-06T14:05:14.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 10 client.0 _ll_register_callbacks cb 0x5593fb11a420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40154/0 (socket says 172.21.15.90:40154) 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 -- 172.21.15.90:0/377114172 learned_addr learned my addr 172.21.15.90:0/377114172 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb234ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 -- 172.21.15.90:0/377114172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 msgr2=0x5593fb234ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb234ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 -- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 msgr2=0x5593fb22db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb22db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 -- 172.21.15.90:0/377114172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f94d8004130 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb234ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94e659c700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f94d8003930 tx=0x7f94d8003a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bbba22c683c013d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94cdffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d8005440 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f94d80055a0 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.948+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5593fb135330 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7f94d801e7a0 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 10 client.15225 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x5593fafa2b10 con 0x5593fb223a00 2024-04-06T14:05:14.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5593fb135330 con 0x5593fb223a00 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f94d803c070 con 0x5593fb223a00 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 10 client.15225 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 10 client.15225 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 20 client.15225 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb238780 0x5593fb23daa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 4 client.15225 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5593fb16d6b0 con 0x5593fb238780 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.949+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb238780 0x5593fb23daa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94cdffb700 10 client.15225 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb238780 0x5593fb23daa0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=2cd554c1a00a35b8 server_cookie=cf72c13db011b98e in_seq=0 out_seq=0 2024-04-06T14:05:14.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94cdffb700 1 -- 172.21.15.90:0/377114172 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5593fb16d6b0 con 0x5593fb238780 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94cdffb700 10 client.15225 handle_command_reply: tid=0 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 client.15225 shutdown 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 msgr2=0x5593fb22d620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f94d8003930 tx=0x7f94d8003a10 comp rx=0 tx=0).stop 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 shutdown_connections 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb238780 0x5593fb23daa0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb224f50 0x5593fb234ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb223a00 0x5593fb22d620 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94cdffb700 0 client.15225 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:05:14.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/377114172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb22db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 >> 172.21.15.90:0/377114172 conn(0x5593fb182650 msgr2=0x5593fb21dbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 shutdown_connections 2024-04-06T14:05:14.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.950+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/377114172 wait complete. 2024-04-06T14:05:14.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.951+0000 7f94f8c4ab80 20 client.15225 trim_cache size 0 max 16384 2024-04-06T14:05:14.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 Processor -- start 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 -- start start 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb223de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224f50 0x5593fb21e530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb16d990 con 0x5593fb224360 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb171850 con 0x5593fb224f50 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb113de0 con 0x5593fb223a00 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.968+0000 7f94e5d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40158/0 (socket says 172.21.15.90:40158) 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 -- 172.21.15.90:0/4090400678 learned_addr learned my addr 172.21.15.90:0/4090400678 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224f50 0x5593fb21e530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e659c700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb223de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 -- 172.21.15.90:0/4090400678 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 msgr2=0x5593fb223de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb223de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 -- 172.21.15.90:0/4090400678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224f50 msgr2=0x5593fb21e530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224f50 0x5593fb21e530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 -- 172.21.15.90:0/4090400678 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5593fb13ae50 con 0x5593fb224360 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e659c700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb223de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f94d00228b0 tx=0x7f94d0022ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8310fd9314add77a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94dd7fa700 1 -- 172.21.15.90:0/4090400678 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d00b0070 con 0x5593fb224360 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94dd7fa700 1 -- 172.21.15.90:0/4090400678 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f94d009b2f0 con 0x5593fb224360 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.969+0000 7f94dd7fa700 1 -- 172.21.15.90:0/4090400678 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94d00cbea0 con 0x5593fb224360 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4090400678 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 msgr2=0x5593fb225330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f94d00228b0 tx=0x7f94d0022ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4090400678 shutdown_connections 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224f50 0x5593fb21e530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb223de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4090400678 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224360 0x5593fb225330 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4090400678 >> 172.21.15.90:0/4090400678 conn(0x5593fb182650 msgr2=0x5593fb22b500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4090400678 shutdown_connections 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4090400678 wait complete. 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- start start 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 Processor -- start 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb2898a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb289de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb291290 con 0x5593fb224f50 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb2913d0 con 0x5593fb224360 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5593fb291560 con 0x5593fb223a00 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94f8c4ab80 10 client.0 _ll_register_callbacks cb 0x5593fb11a420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.970+0000 7f94e6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40168/0 (socket says 172.21.15.90:40168) 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 -- 172.21.15.90:0/4166454136 learned_addr learned my addr 172.21.15.90:0/4166454136 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e659c700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb2898a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb289de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 -- 172.21.15.90:0/4166454136 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 msgr2=0x5593fb2898a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb2898a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 -- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 msgr2=0x5593fb289de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:14.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb289de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 -- 172.21.15.90:0/4166454136 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f94e819f1d0 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e5d9b700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb289de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94e6d9d700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f94e0008ed0 tx=0x7f94e0008f00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7c8681ecd3911bb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94cdffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f94e0003a00 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f94e0003b60 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f94d8003d40 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7f94e0003d20 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94f8c4ab80 10 client.15231 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x5593faffac80 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.971+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f94e0026020 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f94e0015070 con 0x5593fb224f50 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 10 client.15231 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:14.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 10 client.15231 resolve_mds: resolved name 'h' to daemon mds.h 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 20 client.15231 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb296460 0x5593fb292db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 4 client.15231 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5593fafc3ca0 con 0x5593fb296460 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94e659c700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb296460 0x5593fb292db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94e659c700 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb296460 0x5593fb292db0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=10aab5312c10c5e9 server_cookie=a567be7ee67a82eb in_seq=0 out_seq=0 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94cdffb700 10 client.15231 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94cdffb700 1 -- 172.21.15.90:0/4166454136 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x5593fafc3ca0 con 0x5593fb296460 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94cdffb700 10 client.15231 handle_command_reply: tid=0 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.972+0000 7f94f8c4ab80 1 client.15231 shutdown 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 msgr2=0x5593fb290d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f94e0008ed0 tx=0x7f94e0008f00 comp rx=0 tx=0).stop 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 shutdown_connections 2024-04-06T14:05:14.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5593fb296460 0x5593fb292db0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5593fb224f50 0x5593fb290d50 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94cdffb700 0 client.15231 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5593fb223a00 0x5593fb2898a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 --2- 172.21.15.90:0/4166454136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5593fb224360 0x5593fb289de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 >> 172.21.15.90:0/4166454136 conn(0x5593fb182650 msgr2=0x5593fb22d3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 shutdown_connections 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 1 -- 172.21.15.90:0/4166454136 wait complete. 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94f8c4ab80 20 client.15231 trim_cache size 0 max 16384 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:05:14.981 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 2, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24485, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 69.469165743999994, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 83, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 86.534968340999995 2024-04-06T14:05:14.982 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 msgr2=0x7f94d40a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 0x7f94d40a3670 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f94d8005fd0 tx=0x7f94d8005dd0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 msgr2=0x7f94e81972a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f94d0009f50 tx=0x7f94d0003970 comp rx=0 tx=0).stop 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 shutdown_connections 2024-04-06T14:05:14.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f94d40a11b0 0x7f94d40a3670 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f94e8101800 0x7f94e819e280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f94e80ffc70 0x7f94e8196d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 --2- 172.21.15.90:0/42161348 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f94e8100e60 0x7f94e81972a0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 >> 172.21.15.90:0/42161348 conn(0x7f94e80fb200 msgr2=0x7f94e80fcd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 shutdown_connections 2024-04-06T14:05:14.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:14.973+0000 7f94eca96700 1 -- 172.21.15.90:0/42161348 wait complete. 2024-04-06T14:05:14.985 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.i status 2024-04-06T14:05:14.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:14 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3317354324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:14 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3683821243' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:14.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:14 smithi105 ceph-mon[46969]: pgmap v258: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 6 op/s; 58 B/s, 0 objects/s recovering 2024-04-06T14:05:14.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:14 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2092231952' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:15.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 -- 172.21.15.90:0/362942066 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 msgr2=0x7fe5d4109250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/362942066 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d4109250 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7fe5cc009990 tx=0x7fe5cc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 -- 172.21.15.90:0/362942066 shutdown_connections 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/362942066 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d4109250 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/362942066 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d40ff030 0x7fe5d40ff450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/362942066 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d4102000 0x7fe5d4102460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 -- 172.21.15.90:0/362942066 >> 172.21.15.90:0/362942066 conn(0x7fe5d40fa960 msgr2=0x7fe5d40fcda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 -- 172.21.15.90:0/362942066 shutdown_connections 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 -- 172.21.15.90:0/362942066 wait complete. 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.108+0000 7fe5d9e69700 1 Processor -- start 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 -- start start 2024-04-06T14:05:15.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 0x7fe5d4101920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 0x7fe5d40fffb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5d4100c90 con 0x7fe5d41029a0 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5d4100e00 con 0x7fe5d40ff030 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d9e69700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5d41a0ca0 con 0x7fe5d4102000 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40190/0 (socket says 172.21.15.90:40190) 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 -- 172.21.15.90:0/299759735 learned_addr learned my addr 172.21.15.90:0/299759735 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 0x7fe5d40fffb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 0x7fe5d4101920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 -- 172.21.15.90:0/299759735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 msgr2=0x7fe5d40fffb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 0x7fe5d40fffb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 -- 172.21.15.90:0/299759735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 msgr2=0x7fe5d4101920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 0x7fe5d4101920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 -- 172.21.15.90:0/299759735 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5cc0095f0 con 0x7fe5d41029a0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 0x7fe5d4101920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 0x7fe5d40fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.109+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fe5cc0095c0 tx=0x7fe5cc00f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.110+0000 7fe5d0ff9700 1 -- 172.21.15.90:0/299759735 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5cc01d070 con 0x7fe5d41029a0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.110+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5d41a0ed0 con 0x7fe5d41029a0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.110+0000 7fe5d0ff9700 1 -- 172.21.15.90:0/299759735 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe5cc00fb80 con 0x7fe5d41029a0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.110+0000 7fe5d0ff9700 1 -- 172.21.15.90:0/299759735 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5cc017650 con 0x7fe5d41029a0 2024-04-06T14:05:15.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.110+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5d41a1290 con 0x7fe5d41029a0 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.112+0000 7fe5d0ff9700 1 -- 172.21.15.90:0/299759735 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fe5cc022470 con 0x7fe5d41029a0 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.112+0000 7fe5d0ff9700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 0x7fe5bc0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.112+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 0x7fe5bc0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.113+0000 7fe5d0ff9700 1 -- 172.21.15.90:0/299759735 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fe5cc0cdf90 con 0x7fe5d41029a0 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.113+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 0x7fe5bc0a3670 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe5d4067c00 tx=0x7fe5c4009040 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.113+0000 7fe5e601db80 1 Processor -- start 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.113+0000 7fe5e601db80 1 -- start start 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x563423506970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x5634235073b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56342345d940 con 0x563423505a00 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56342345b300 con 0x563423506f50 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5e601db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563423456bf0 con 0x563423506360 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40198/0 (socket says 172.21.15.90:40198) 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 -- 172.21.15.90:0/4273976670 learned_addr learned my addr 172.21.15.90:0/4273976670 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x5634235073b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 -- 172.21.15.90:0/4273976670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 msgr2=0x563423506970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x563423506970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x563423506970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 -- 172.21.15.90:0/4273976670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 msgr2=0x5634235073b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x5634235073b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 -- 172.21.15.90:0/4273976670 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5634232d8360 con 0x563423505a00 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x5634235073b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fe5c4018540 tx=0x7fe5c4011000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bd92063f856b1c7f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5b9ffb700 1 -- 172.21.15.90:0/4273976670 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5c401d070 con 0x563423505a00 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5b9ffb700 1 -- 172.21.15.90:0/4273976670 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe5c40180d0 con 0x563423505a00 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.114+0000 7fe5b9ffb700 1 -- 172.21.15.90:0/4273976670 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5c4016ea0 con 0x563423505a00 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- 172.21.15.90:0/4273976670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 msgr2=0x563423505e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7fe5c4018540 tx=0x7fe5c4011000 comp rx=0 tx=0).stop 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- 172.21.15.90:0/4273976670 shutdown_connections 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x5634235073b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423505e20 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 --2- 172.21.15.90:0/4273976670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x563423506970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- 172.21.15.90:0/4273976670 >> 172.21.15.90:0/4273976670 conn(0x563423464650 msgr2=0x563423500560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- 172.21.15.90:0/4273976670 shutdown_connections 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- 172.21.15.90:0/4273976670 wait complete. 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.115+0000 7fe5e601db80 1 -- start start 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 Processor -- start 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423505a00 0x56342350f660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x563423516be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634233f2d30 con 0x563423506360 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634233f29d0 con 0x563423506f50 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634232da650 con 0x563423505a00 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5e601db80 10 client.0 _ll_register_callbacks cb 0x5634233fc420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40206/0 (socket says 172.21.15.90:40206) 2024-04-06T14:05:15.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 -- 172.21.15.90:0/1025953147 learned_addr learned my addr 172.21.15.90:0/1025953147 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x563423516be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423505a00 0x56342350f660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 -- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423505a00 msgr2=0x56342350f660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423505a00 0x56342350f660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 -- 172.21.15.90:0/1025953147 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 msgr2=0x563423516be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x563423516be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.116+0000 7fe5d2ffd700 1 -- 172.21.15.90:0/1025953147 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5c4007dc0 con 0x563423506360 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fe5c800f480 tx=0x7fe5c800f790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=480e5454294cacca server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5c8011850 con 0x563423506360 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fe5c8015070 con 0x563423506360 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563423417330 con 0x563423506360 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe5c800fec0 con 0x563423506360 2024-04-06T14:05:15.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 10 client.15243 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x563423284b10 con 0x563423506360 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563423417330 con 0x563423506360 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe5c8023070 con 0x563423506360 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 10 client.15243 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 10 client.15243 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 20 client.15243 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x56342351fab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 4 client.15243 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56342344f6b0 con 0x56342351a7f0 2024-04-06T14:05:15.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.117+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x56342351fab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.118+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x56342351fab0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=667ab4aa69ed2a26 server_cookie=ed7909e415104da1 in_seq=0 out_seq=0 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.118+0000 7fe5b2ffd700 10 client.15243 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/1025953147 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56342344f6b0 con 0x56342351a7f0 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5b2ffd700 10 client.15243 handle_command_reply: tid=0 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 client.15243 shutdown 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 msgr2=0x56342350fba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7fe5c800f480 tx=0x7fe5c800f790 comp rx=0 tx=0).stop 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 shutdown_connections 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x56342351fab0 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506f50 0x563423516be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5b2ffd700 0 client.15243 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423505a00 0x56342350f660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 --2- 172.21.15.90:0/1025953147 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506360 0x56342350fba0 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 >> 172.21.15.90:0/1025953147 conn(0x563423464650 msgr2=0x5634234ffbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 shutdown_connections 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 1 -- 172.21.15.90:0/1025953147 wait complete. 2024-04-06T14:05:15.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.119+0000 7fe5e601db80 20 client.15243 trim_cache size 0 max 16384 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.136+0000 7fe5e601db80 1 Processor -- start 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 -- start start 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 0x563423517290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x56342350eb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56342344f990 con 0x563423506f50 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563423453850 con 0x563423505a00 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5e601db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634233f5de0 con 0x563423506360 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40218/0 (socket says 172.21.15.90:40218) 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 -- 172.21.15.90:0/3521637114 learned_addr learned my addr 172.21.15.90:0/3521637114 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 0x563423517290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x56342350eb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 -- 172.21.15.90:0/3521637114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 msgr2=0x56342350eb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x56342350eb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 -- 172.21.15.90:0/3521637114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 msgr2=0x563423517290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 0x563423517290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 -- 172.21.15.90:0/3521637114 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56342341ce50 con 0x563423506f50 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 0x563423517290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fe5cc005d90 tx=0x7fe5cc01a040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cf8e5893567c7a87 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5ba7fc700 1 -- 172.21.15.90:0/3521637114 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5cc0a1040 con 0x563423506f50 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5ba7fc700 1 -- 172.21.15.90:0/3521637114 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe5cc022e90 con 0x563423506f50 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.137+0000 7fe5ba7fc700 1 -- 172.21.15.90:0/3521637114 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5cc09a8d0 con 0x563423506f50 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- 172.21.15.90:0/3521637114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 msgr2=0x563423516ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fe5cc005d90 tx=0x7fe5cc01a040 comp rx=0 tx=0).stop 2024-04-06T14:05:15.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- 172.21.15.90:0/3521637114 shutdown_connections 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423506f50 0x563423516ba0 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423505a00 0x563423517290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 --2- 172.21.15.90:0/3521637114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506360 0x56342350eb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- 172.21.15.90:0/3521637114 >> 172.21.15.90:0/3521637114 conn(0x563423464650 msgr2=0x56342350f660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- 172.21.15.90:0/3521637114 shutdown_connections 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- 172.21.15.90:0/3521637114 wait complete. 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.138+0000 7fe5e601db80 1 -- start start 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 Processor -- start 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 0x563423514000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 0x56342356f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5634235146e0 con 0x563423505a00 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56342356f5e0 con 0x563423506360 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56342356f770 con 0x563423506f50 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 10 client.0 _ll_register_callbacks cb 0x5634233fc420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40220/0 (socket says 172.21.15.90:40220) 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 -- 172.21.15.90:0/189562261 learned_addr learned my addr 172.21.15.90:0/189562261 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 0x563423514000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 0x56342356f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 -- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 msgr2=0x56342356f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 0x56342356f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 -- 172.21.15.90:0/189562261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 msgr2=0x563423514000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 0x563423514000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 -- 172.21.15.90:0/189562261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5d41a1290 con 0x563423505a00 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d3fff700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 0x56342356f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 0x563423514000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5d37fe700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fe5c4018510 tx=0x7fe5c4018240 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c36cd97130c00d4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5b2ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe5c401d040 con 0x563423505a00 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fe5c4004c70 con 0x563423505a00 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.139+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe5c40180d0 con 0x563423505a00 2024-04-06T14:05:15.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe5c4016ea0 con 0x563423505a00 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 10 client.15249 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x5634232dcc80 con 0x563423505a00 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe5c40180d0 con 0x563423505a00 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe5c402e100 con 0x563423505a00 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 10 client.15249 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 10 client.15249 resolve_mds: resolved name 'i' to daemon mds.i 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 20 client.15249 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x563423570f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 4 client.15249 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5634232a5ca0 con 0x56342351a7f0 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.140+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x563423570f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5b2ffd700 10 client.15249 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5d2ffd700 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x563423570f80 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ebf0a9755a4be09e server_cookie=775b5e34c1a01cff in_seq=0 out_seq=0 2024-04-06T14:05:15.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5b2ffd700 1 -- 172.21.15.90:0/189562261 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x5634232a5ca0 con 0x56342351a7f0 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5b2ffd700 10 client.15249 handle_command_reply: tid=0 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 client.15249 shutdown 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 msgr2=0x563423513ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fe5c4018510 tx=0x7fe5c4018240 comp rx=0 tx=0).stop 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 shutdown_connections 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56342351a7f0 0x563423570f80 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563423506f50 0x56342356f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5b2ffd700 0 client.15249 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563423505a00 0x563423513ac0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 --2- 172.21.15.90:0/189562261 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563423506360 0x563423514000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 >> 172.21.15.90:0/189562261 conn(0x563423464650 msgr2=0x563423523ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 shutdown_connections 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 1 -- 172.21.15.90:0/189562261 wait complete. 2024-04-06T14:05:15.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.141+0000 7fe5e601db80 20 client.15249 trim_cache size 0 max 16384 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 1, 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24490, 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 70.643800087000002, 2024-04-06T14:05:15.150 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:05:15.151 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 83, 2024-04-06T14:05:15.151 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:05:15.151 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 84.982218082000003 2024-04-06T14:05:15.151 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:05:15.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 msgr2=0x7fe5bc0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 0x7fe5bc0a3670 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fe5d4067c00 tx=0x7fe5c4009040 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 msgr2=0x7fe5d41004f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7fe5cc0095c0 tx=0x7fe5cc00f690 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 shutdown_connections 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fe5bc0a11b0 0x7fe5bc0a3670 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5d41029a0 0x7fe5d41004f0 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5d40ff030 0x7fe5d4101920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 --2- 172.21.15.90:0/299759735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5d4102000 0x7fe5d40fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 >> 172.21.15.90:0/299759735 conn(0x7fe5d40fa960 msgr2=0x7fe5d4105350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 shutdown_connections 2024-04-06T14:05:15.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.142+0000 7fe5d9e69700 1 -- 172.21.15.90:0/299759735 wait complete. 2024-04-06T14:05:15.153 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 -- 172.21.15.90:0/2794087490 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8102920 msgr2=0x7f80d81091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/2794087490 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8102920 0x7f80d81091d0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f80d000b0d0 tx=0x7f80d000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 -- 172.21.15.90:0/2794087490 shutdown_connections 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/2794087490 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8102920 0x7f80d81091d0 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/2794087490 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8100d90 0x7f80d81011b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/2794087490 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8101f80 0x7f80d81023e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 -- 172.21.15.90:0/2794087490 >> 172.21.15.90:0/2794087490 conn(0x7f80d80fc300 msgr2=0x7f80d80fe740 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 -- 172.21.15.90:0/2794087490 shutdown_connections 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.277+0000 7f80dfe5f700 1 -- 172.21.15.90:0/2794087490 wait complete. 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 Processor -- start 2024-04-06T14:05:15.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 -- start start 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 0x7f80d81972d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 0x7f80d819e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80d819e8c0 con 0x7f80d8100d90 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80d819ea00 con 0x7f80d8102920 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dfe5f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80d819eb90 con 0x7f80d8101f80 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40238/0 (socket says 172.21.15.90:40238) 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 -- 172.21.15.90:0/3985183576 learned_addr learned my addr 172.21.15.90:0/3985183576 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80de3fc700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 0x7f80d819e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 0x7f80d81972d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 -- 172.21.15.90:0/3985183576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 msgr2=0x7f80d81972d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 0x7f80d81972d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 -- 172.21.15.90:0/3985183576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 msgr2=0x7f80d819e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 0x7f80d819e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80ddbfb700 1 -- 172.21.15.90:0/3985183576 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80d0009d00 con 0x7f80d8100d90 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 0x7f80d81972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.278+0000 7f80de3fc700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 0x7f80d819e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f80cc00d980 tx=0x7f80cc00dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80caffd700 1 -- 172.21.15.90:0/3985183576 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80cc009b40 con 0x7f80d8100d90 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80caffd700 1 -- 172.21.15.90:0/3985183576 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80cc00de60 con 0x7f80d8100d90 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80d819ee70 con 0x7f80d8100d90 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80caffd700 1 -- 172.21.15.90:0/3985183576 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80cc00f620 con 0x7f80d8100d90 2024-04-06T14:05:15.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.279+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80d819f290 con 0x7f80d8100d90 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.281+0000 7f80caffd700 1 -- 172.21.15.90:0/3985183576 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7f80cc010460 con 0x7f80d8100d90 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.281+0000 7f80caffd700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 0x7f80c40a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.281+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 0x7f80c40a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.281+0000 7f80caffd700 1 -- 172.21.15.90:0/3985183576 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f80cc0cc5d0 con 0x7f80d8100d90 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.282+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 0x7f80c40a36f0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f80d4005fd0 tx=0x7f80d4005e20 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.282+0000 7f80ec013b80 1 Processor -- start 2024-04-06T14:05:15.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.282+0000 7f80ec013b80 1 -- start start 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x55845305de20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x55845305e970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452fb5940 con 0x55845305ef50 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452fb3300 con 0x55845305e360 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ec013b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452faebf0 con 0x55845305da00 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40254/0 (socket says 172.21.15.90:40254) 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 -- 172.21.15.90:0/2811579979 learned_addr learned my addr 172.21.15.90:0/2811579979 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x55845305de20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x55845305e970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 -- 172.21.15.90:0/2811579979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 msgr2=0x55845305de20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x55845305de20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 -- 172.21.15.90:0/2811579979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 msgr2=0x55845305e970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x55845305e970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 -- 172.21.15.90:0/2811579979 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558452e30360 con 0x55845305ef50 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x55845305e970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f80d0007bd0 tx=0x7f80d0009c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e136d9a8ab3fdd5b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80c3fff700 1 -- 172.21.15.90:0/2811579979 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80d000e070 con 0x55845305ef50 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80c3fff700 1 -- 172.21.15.90:0/2811579979 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80d00090d0 con 0x55845305ef50 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.283+0000 7f80c3fff700 1 -- 172.21.15.90:0/2811579979 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80d0005040 con 0x55845305ef50 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- 172.21.15.90:0/2811579979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 msgr2=0x55845305f3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f80d0007bd0 tx=0x7f80d0009c20 comp rx=0 tx=0).stop 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- 172.21.15.90:0/2811579979 shutdown_connections 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x55845305f3b0 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x55845305de20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2811579979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x55845305e970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- 172.21.15.90:0/2811579979 >> 172.21.15.90:0/2811579979 conn(0x558452fbc650 msgr2=0x558453058560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- 172.21.15.90:0/2811579979 shutdown_connections 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- 172.21.15.90:0/2811579979 wait complete. 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.284+0000 7f80ec013b80 1 -- start start 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 Processor -- start 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x558453067b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845306eb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452f4ad30 con 0x55845305da00 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452f4a9d0 con 0x55845305ef50 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452e32650 con 0x55845305e360 2024-04-06T14:05:15.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ec013b80 10 client.0 _ll_register_callbacks cb 0x558452f54420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40258/0 (socket says 172.21.15.90:40258) 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 -- 172.21.15.90:0/2858140211 learned_addr learned my addr 172.21.15.90:0/2858140211 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845306eb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x558453067b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 -- 172.21.15.90:0/2858140211 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 msgr2=0x558453067b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x558453067b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 -- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 msgr2=0x55845306eb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845306eb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80ddbfb700 1 -- 172.21.15.90:0/2858140211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80d8109b20 con 0x55845305da00 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.285+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845306eb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f80cc096340 tx=0x7f80cc09a110 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ee51f8bfd5ffee18 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:15.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80cc0a1d50 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f80cc099b30 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558452f6f330 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7f80cc0b2200 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 10 client.15261 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x558452ddcb10 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x558452f6f330 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f80cc0a18c0 con 0x55845305da00 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 10 client.15261 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 10 client.15261 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 20 client.15261 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530720a0 0x558453077990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 4 client.15261 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558452fa76b0 con 0x5584530720a0 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.286+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530720a0 0x558453077990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.287+0000 7f80c17fa700 10 client.15261 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.287+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530720a0 0x558453077990 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=217413f311c3c2db server_cookie=2d5924b3398ce68e in_seq=0 out_seq=0 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.287+0000 7f80c17fa700 1 -- 172.21.15.90:0/2858140211 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x558452fa76b0 con 0x5584530720a0 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.287+0000 7f80c17fa700 10 client.15261 handle_command_reply: tid=0 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 client.15261 shutdown 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 msgr2=0x558453067610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7f80cc096340 tx=0x7f80cc09a110 comp rx=0 tx=0).stop 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 shutdown_connections 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845306eb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x558453067610 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530720a0 0x558453077990 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2858140211 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x558453067b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80c17fa700 0 client.15261 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:15.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 >> 172.21.15.90:0/2858140211 conn(0x558452fbc650 msgr2=0x558453057bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 shutdown_connections 2024-04-06T14:05:15.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 1 -- 172.21.15.90:0/2858140211 wait complete. 2024-04-06T14:05:15.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.288+0000 7f80ec013b80 20 client.15261 trim_cache size 0 max 16384 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.305+0000 7f80ec013b80 1 Processor -- start 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.305+0000 7f80ec013b80 1 -- start start 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.305+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x55845306f240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845307b9b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ec013b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452f6f330 con 0x55845305da00 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ec013b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558453065ba0 con 0x55845305ef50 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ec013b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558452f6cad0 con 0x55845305e360 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40264/0 (socket says 172.21.15.90:40264) 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 -- 172.21.15.90:0/929346627 learned_addr learned my addr 172.21.15.90:0/929346627 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x55845306f240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80de3fc700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845307b9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 -- 172.21.15.90:0/929346627 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 msgr2=0x55845306f240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x55845306f240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 -- 172.21.15.90:0/929346627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 msgr2=0x55845307b9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845307b9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 -- 172.21.15.90:0/929346627 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558452f74e50 con 0x55845305da00 2024-04-06T14:05:15.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80de3fc700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845307b9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f80cc09a180 tx=0x7f80cc0997d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0b23ae027a95d5e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80c8ff9700 1 -- 172.21.15.90:0/929346627 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80cc099410 con 0x55845305da00 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80c8ff9700 1 -- 172.21.15.90:0/929346627 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80cc0b2200 con 0x55845305da00 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.306+0000 7f80c8ff9700 1 -- 172.21.15.90:0/929346627 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80cc098dc0 con 0x55845305da00 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- 172.21.15.90:0/929346627 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 msgr2=0x55845305dde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f80cc09a180 tx=0x7f80cc0997d0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- 172.21.15.90:0/929346627 shutdown_connections 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305ef50 0x55845307b9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305da00 0x55845305dde0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 --2- 172.21.15.90:0/929346627 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305e360 0x55845306f240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- 172.21.15.90:0/929346627 >> 172.21.15.90:0/929346627 conn(0x558452fbc650 msgr2=0x558453065490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- 172.21.15.90:0/929346627 shutdown_connections 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- 172.21.15.90:0/929346627 wait complete. 2024-04-06T14:05:15.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 -- start start 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.307+0000 7f80ec013b80 1 Processor -- start 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x5584530c3350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x5584530c3890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5584530cae10 con 0x55845305ef50 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5584530caf50 con 0x55845305e360 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5584530cb0e0 con 0x55845305da00 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 10 client.0 _ll_register_callbacks cb 0x558452f54420 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40278/0 (socket says 172.21.15.90:40278) 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 -- 172.21.15.90:0/2033082598 learned_addr learned my addr 172.21.15.90:0/2033082598 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x5584530c3350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80dd3fa700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x5584530c3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 -- 172.21.15.90:0/2033082598 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 msgr2=0x5584530c3350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x5584530c3350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 -- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 msgr2=0x5584530c3890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x5584530c3890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 -- 172.21.15.90:0/2033082598 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558452ddcb10 con 0x55845305ef50 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80de3fc700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f80d000b010 tx=0x7f80d00076d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fb9c4b0a3597e4c3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80c17fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80d000e050 con 0x55845305ef50 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f80d002a070 con 0x55845305ef50 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.308+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f80d00090d0 con 0x55845305ef50 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=23) v2 ==== 24+0+0 (secure 0 0 0) 0x7f80d0006000 con 0x55845305ef50 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 10 client.15267 fetch_fsmap learned FSMap version 23 2024-04-06T14:05:15.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=23,monmap=4+}) v3 -- 0x558452e34c80 con 0x55845305ef50 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80d00090d0 con 0x55845305ef50 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 23) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f80d000b100 con 0x55845305ef50 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 10 client.15267 fetch_fsmap finished waiting for FSMap version 23 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 10 client.15267 resolve_mds: resolved name 'b' to daemon mds.b 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 20 client.15267 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530cfbc0 0x5584530cc5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 4 client.15267 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "status"}] 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "status"}) v1 -- 0x558452dfdca0 con 0x5584530cfbc0 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.309+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530cfbc0 0x5584530cc5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80c17fa700 10 client.15267 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ddbfb700 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530cfbc0 0x5584530cc5f0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=15bdd12f6aea1e2f server_cookie=7428fac70c2e11eb in_seq=0 out_seq=0 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80c17fa700 1 -- 172.21.15.90:0/2033082598 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x558452dfdca0 con 0x5584530cfbc0 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80c17fa700 10 client.15267 handle_command_reply: tid=0 2024-04-06T14:05:15.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 client.15267 shutdown 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 msgr2=0x5584530ca8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f80d000b010 tx=0x7f80d00076d0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 shutdown_connections 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55845305ef50 0x5584530ca8d0 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5584530cfbc0 0x5584530cc5f0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55845305da00 0x5584530c3350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 --2- 172.21.15.90:0/2033082598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55845305e360 0x5584530c3890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80c17fa700 0 client.15267 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 >> 172.21.15.90:0/2033082598 conn(0x558452fbc650 msgr2=0x558453061ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 shutdown_connections 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 1 -- 172.21.15.90:0/2033082598 wait complete. 2024-04-06T14:05:15.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.310+0000 7f80ec013b80 20 client.15267 trim_cache size 0 max 16384 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "cluster_fsid": "949dca1a-f41d-11ee-b649-cb9ed24678a4", 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "whoami": 0, 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "id": 24515, 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "want_state": "up:active", 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "state": "up:active", 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "fs_name": "cephfs", 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "rank_uptime": 73.876324608000004, 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "mdsmap_epoch": 23, 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch": 83, 2024-04-06T14:05:15.317 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_epoch_barrier": 81, 2024-04-06T14:05:15.318 INFO:teuthology.orchestra.run.smithi090.stdout: "uptime": 79.881147443000003 2024-04-06T14:05:15.318 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:05:15.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 msgr2=0x7f80c40a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 0x7f80c40a36f0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f80d4005fd0 tx=0x7f80d4005e20 comp rx=0 tx=0).stop 2024-04-06T14:05:15.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 msgr2=0x7f80d8196d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f80cc00d980 tx=0x7f80cc00dd40 comp rx=0 tx=0).stop 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 shutdown_connections 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f80c40a1230 0x7f80c40a36f0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80d8102920 0x7f80d819e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f80d8100d90 0x7f80d8196d90 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 --2- 172.21.15.90:0/3985183576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80d8101f80 0x7f80d81972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 >> 172.21.15.90:0/3985183576 conn(0x7f80d80fc300 msgr2=0x7f80d8105220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 shutdown_connections 2024-04-06T14:05:15.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.311+0000 7f80dfe5f700 1 -- 172.21.15.90:0/3985183576 wait complete. 2024-04-06T14:05:15.322 INFO:teuthology.run_tasks:Running task sequential... 2024-04-06T14:05:15.332 INFO:teuthology.task.sequential:In sequential, running task sequential... 2024-04-06T14:05:15.332 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-06T14:05:15.332 INFO:teuthology.task.print:Enabling mgr modules 2024-04-06T14:05:15.333 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-04-06T14:05:15.333 INFO:teuthology.task.exec:Executing custom commands... 2024-04-06T14:05:15.333 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:05:15.334 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph mgr module enable snap_schedule' 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- 172.21.15.90:0/1409122728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd244102940 msgr2=0x7fd2441091f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 --2- 172.21.15.90:0/1409122728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd244102940 0x7fd2441091f0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fd23c00b0a0 tx=0x7fd23c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- 172.21.15.90:0/1409122728 shutdown_connections 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 --2- 172.21.15.90:0/1409122728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd244102940 0x7fd2441091f0 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 --2- 172.21.15.90:0/1409122728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd2440fefd0 0x7fd2440ff3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 --2- 172.21.15.90:0/1409122728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd244102400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- 172.21.15.90:0/1409122728 >> 172.21.15.90:0/1409122728 conn(0x7fd2440fa900 msgr2=0x7fd2440fcd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- 172.21.15.90:0/1409122728 shutdown_connections 2024-04-06T14:05:15.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- 172.21.15.90:0/1409122728 wait complete. 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 Processor -- start 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.456+0000 7fd24a378700 1 -- start start 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd2441972a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd244102940 0x7fd24419e280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd24419e850 con 0x7fd2440fefd0 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd24419e9c0 con 0x7fd244101fa0 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd24419eb80 con 0x7fd244102940 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40288/0 (socket says 172.21.15.90:40288) 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 -- 172.21.15.90:0/1221283268 learned_addr learned my addr 172.21.15.90:0/1221283268 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd2437fe700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd2441972a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd248915700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd244102940 0x7fd24419e280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 -- 172.21.15.90:0/1221283268 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd244102940 msgr2=0x7fd24419e280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd244102940 0x7fd24419e280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 -- 172.21.15.90:0/1221283268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 msgr2=0x7fd2441972a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd2441972a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 -- 172.21.15.90:0/1221283268 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd23c009d00 con 0x7fd2440fefd0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd2437fe700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd2441972a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd243fff700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fd23800e970 tx=0x7fd23800ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd238009790 con 0x7fd2440fefd0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd24419ee60 con 0x7fd2440fefd0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd2380044e0 con 0x7fd2440fefd0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.457+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd238003e30 con 0x7fd2440fefd0 2024-04-06T14:05:15.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.458+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd24419f290 con 0x7fd2440fefd0 2024-04-06T14:05:15.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.459+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd244108160 con 0x7fd2440fefd0 2024-04-06T14:05:15.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.460+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 17) v1 ==== 135924+0+0 (secure 0 0 0) 0x7fd238004000 con 0x7fd2440fefd0 2024-04-06T14:05:15.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.460+0000 7fd2417fa700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 0x7fd22c0a3670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:15.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.460+0000 7fd2437fe700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 0x7fd22c0a3670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:15.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.460+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7fd238014070 con 0x7fd2440fefd0 2024-04-06T14:05:15.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.461+0000 7fd2437fe700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 0x7fd22c0a3670 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fd234005fd0 tx=0x7fd234005e30 comp rx=0 tx=0).ready entity=mgr.14152 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:15.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.464+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd2380963c0 con 0x7fd2440fefd0 2024-04-06T14:05:15.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:15.720+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "snap_schedule"} v 0) v1 -- 0x7fd24410c810 con 0x7fd2440fefd0 2024-04-06T14:05:15.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:15 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/859122633' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:15.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:15 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/859122633' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:15.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:15 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/859122633' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.549+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mgrmap(e 18) v1 ==== 135941+0+0 (secure 0 0 0) 0x7fd238096790 con 0x7fd2440fefd0 2024-04-06T14:05:16.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.550+0000 7fd2417fa700 1 -- 172.21.15.90:0/1221283268 <== mon.0 v2:172.21.15.90:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "snap_schedule"}]=0 v18) v1 ==== 92+0+0 (secure 0 0 0) 0x7fd2380963c0 con 0x7fd2440fefd0 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 msgr2=0x7fd22c0a3670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 0x7fd22c0a3670 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fd234005fd0 tx=0x7fd234005e30 comp rx=0 tx=0).stop 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 msgr2=0x7fd244196d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7fd23800e970 tx=0x7fd23800ed30 comp rx=0 tx=0).stop 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 shutdown_connections 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7fd22c0a11b0 0x7fd22c0a3670 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd244102940 0x7fd24419e280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd2440fefd0 0x7fd244196d60 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 --2- 172.21.15.90:0/1221283268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd244101fa0 0x7fd2441972a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 >> 172.21.15.90:0/1221283268 conn(0x7fd2440fa900 msgr2=0x7fd2441052a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:16.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 shutdown_connections 2024-04-06T14:05:16.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.553+0000 7fd24a378700 1 -- 172.21.15.90:0/1221283268 wait complete. 2024-04-06T14:05:16.567 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/allow_m_granularity true' 2024-04-06T14:05:16.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.712+0000 7f9c242e3700 1 -- 172.21.15.90:0/438234338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c1017b0 msgr2=0x7f9c1c108060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- 172.21.15.90:0/438234338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c108060 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f9c1400b0d0 tx=0x7f9c1400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- 172.21.15.90:0/438234338 shutdown_connections 2024-04-06T14:05:16.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- 172.21.15.90:0/438234338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c108060 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- 172.21.15.90:0/438234338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c100040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- 172.21.15.90:0/438234338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c100e10 0x7f9c1c101270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- 172.21.15.90:0/438234338 >> 172.21.15.90:0/438234338 conn(0x7f9c1c0fb190 msgr2=0x7f9c1c0fd5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- 172.21.15.90:0/438234338 shutdown_connections 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- 172.21.15.90:0/438234338 wait complete. 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 Processor -- start 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- start start 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c100e10 0x7f9c1c1022c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c102800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c1c102fa0 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c1c103110 con 0x7f9c1c1017b0 2024-04-06T14:05:16.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.713+0000 7f9c242e3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c1c1a0e10 con 0x7f9c1c100e10 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40328/0 (socket says 172.21.15.90:40328) 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 -- 172.21.15.90:0/1251859673 learned_addr learned my addr 172.21.15.90:0/1251859673 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2187e700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c100e10 0x7f9c1c1022c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c22880700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c102800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c100e10 msgr2=0x7f9c1c1022c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c100e10 0x7f9c1c1022c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c1017b0 msgr2=0x7f9c1c102800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c102800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 -- 172.21.15.90:0/1251859673 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c14009d00 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c2207f700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f9c1800e970 tx=0x7f9c1800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c18009790 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9c1c1a10a0 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9c1800c780 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c18003be0 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.714+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c1c1a15b0 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.715+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9c1c0673b0 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.716+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 18) v1 ==== 135941+0+0 (secure 0 0 0) 0x7f9c18003d60 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.717+0000 7f9c137fe700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 0x7f9c080a36c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.717+0000 7f9c2187e700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 msgr2=0x7f9c080a36c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.717+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f9c18014070 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.717+0000 7f9c2187e700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 0x7f9c080a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T14:05:16.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.720+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9c18096440 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:16 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-06T14:05:16.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:16 smithi090 ceph-mon[43420]: pgmap v259: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 5 op/s; 52 B/s, 0 objects/s recovering 2024-04-06T14:05:16.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setuser ceph since I am not root 2024-04-06T14:05:16.789 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: ignoring --setgroup ceph since I am not root 2024-04-06T14:05:16.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-06T14:05:16.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-mon[47549]: pgmap v259: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 5 op/s; 52 B/s, 0 objects/s recovering 2024-04-06T14:05:16.894 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: ignoring --setuser ceph since I am not root 2024-04-06T14:05:16.894 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: ignoring --setgroup ceph since I am not root 2024-04-06T14:05:16.894 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-mgr[49433]: -- 172.21.15.134:0/802491089 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e1e66961a0 con 0x55e1e666cc00 2024-04-06T14:05:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.918+0000 7f9c2187e700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 msgr2=0x7f9c080a36c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.918+0000 7f9c2187e700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 0x7f9c080a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T14:05:16.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.929+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}] v 0) v1 -- 0x7f9c1c10b440 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.933+0000 7f9c137fe700 1 -- 172.21.15.90:0/1251859673 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}]=0 v28)=0 v28) v1 ==== 138+0+0 (secure 0 0 0) 0x7f9c18099a60 con 0x7f9c1c0ffc20 2024-04-06T14:05:16.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 msgr2=0x7f9c080a36c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:05:16.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 0x7f9c080a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 msgr2=0x7f9c1c103c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:16.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f9c1800e970 tx=0x7f9c1800ec80 comp rx=0 tx=0).stop 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 shutdown_connections 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f9c080a1200 0x7f9c080a36c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9c1c1017b0 0x7f9c1c102800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9c1c0ffc20 0x7f9c1c103c30 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 --2- 172.21.15.90:0/1251859673 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9c1c100e10 0x7f9c1c1022c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 >> 172.21.15.90:0/1251859673 conn(0x7f9c1c0fb190 msgr2=0x7f9c1c068fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 shutdown_connections 2024-04-06T14:05:16.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:16.935+0000 7f9c242e3700 1 -- 172.21.15.90:0/1251859673 wait complete. 2024-04-06T14:05:16.947 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/dump_on_update true' 2024-04-06T14:05:16.980 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2024-04-06T14:05:16.981 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-mon[46969]: pgmap v259: 97 pgs: 97 active+clean; 640 KiB data, 351 MiB used, 1.0 TiB / 1.0 TiB avail; 3.2 KiB/s rd, 5 op/s; 52 B/s, 0 objects/s recovering 2024-04-06T14:05:16.981 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: ignoring --setuser ceph since I am not root 2024-04-06T14:05:16.981 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: ignoring --setgroup ceph since I am not root 2024-04-06T14:05:16.981 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-mgr[48476]: -- 172.21.15.105:0/3608232428 <== mon.2 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5619ff6e01a0 con 0x5619ff6b6c00 2024-04-06T14:05:16.981 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:16.979+0000 7f721fb6f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T14:05:17.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- 172.21.15.90:0/3825994177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 msgr2=0x7effe4100290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:17.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 --2- 172.21.15.90:0/3825994177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4100290 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7effd0009990 tx=0x7effd0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- 172.21.15.90:0/3825994177 shutdown_connections 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 --2- 172.21.15.90:0/3825994177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 0x7effe4108070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 --2- 172.21.15.90:0/3825994177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 0x7effe41015e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 --2- 172.21.15.90:0/3825994177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4100290 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- 172.21.15.90:0/3825994177 >> 172.21.15.90:0/3825994177 conn(0x7effe40fa900 msgr2=0x7effe40fcd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- 172.21.15.90:0/3825994177 shutdown_connections 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- 172.21.15.90:0/3825994177 wait complete. 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 Processor -- start 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.074+0000 7effeae94700 1 -- start start 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 0x7effe41971d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 0x7effe419e1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:17.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7effe419e780 con 0x7effe40ffe10 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7effe419e8f0 con 0x7effe41007d0 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7effe419eab0 con 0x7effe41011c0 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40346/0 (socket says 172.21.15.90:40346) 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 -- 172.21.15.90:0/3353489036 learned_addr learned my addr 172.21.15.90:0/3353489036 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe9431700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 0x7effe419e1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe3fff700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 0x7effe41971d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 msgr2=0x7effe419e1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 0x7effe419e1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 msgr2=0x7effe41971d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 0x7effe41971d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 -- 172.21.15.90:0/3353489036 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7effd00095f0 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe8c30700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7effd800d980 tx=0x7effd800dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effd8004220 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7effd8007640 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7effe419ed90 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7effd8005690 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.075+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7effe419f1e0 con 0x7effe40ffe10 2024-04-06T14:05:17.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.076+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7effe40673b0 con 0x7effe40ffe10 2024-04-06T14:05:17.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.077+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 18) v1 ==== 135941+0+0 (secure 0 0 0) 0x7effd8004750 con 0x7effe40ffe10 2024-04-06T14:05:17.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.078+0000 7effe1ffb700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 0x7effd40a36c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:17.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.078+0000 7effe3fff700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 msgr2=0x7effd40a36c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:17.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.078+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7effd80cc850 con 0x7effe40ffe10 2024-04-06T14:05:17.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.078+0000 7effe3fff700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 0x7effd40a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T14:05:17.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.082+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7effd8095e90 con 0x7effe40ffe10 2024-04-06T14:05:17.205 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:17.015+0000 7f923225c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T14:05:17.206 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:17.115+0000 7f923225c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T14:05:17.206 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:17.206+0000 7f923225c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T14:05:17.246 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:17.072+0000 7f721fb6f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T14:05:17.246 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:17.160+0000 7f721fb6f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T14:05:17.254 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:16 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:16.987+0000 7f09e93ef200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-06T14:05:17.254 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:17.079+0000 7f09e93ef200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-06T14:05:17.254 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:17.167+0000 7f09e93ef200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-06T14:05:17.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.279+0000 7effe3fff700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 msgr2=0x7effd40a36c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:17.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.279+0000 7effe3fff700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 0x7effd40a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T14:05:17.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.286+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/dump_on_update}] v 0) v1 -- 0x7effe410b400 con 0x7effe40ffe10 2024-04-06T14:05:17.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.291+0000 7effe1ffb700 1 -- 172.21.15.90:0/3353489036 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/dump_on_update}]=0 v29)=0 v29) v1 ==== 133+0+0 (secure 0 0 0) 0x7effd8016020 con 0x7effe40ffe10 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 msgr2=0x7effd40a36c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 0x7effd40a36c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 msgr2=0x7effe4196c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7effd800d980 tx=0x7effd800dc90 comp rx=0 tx=0).stop 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 shutdown_connections 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7effd40a1200 0x7effd40a36c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7effe41011c0 0x7effe419e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7effe40ffe10 0x7effe4196c90 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 --2- 172.21.15.90:0/3353489036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7effe41007d0 0x7effe41971d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:17.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 >> 172.21.15.90:0/3353489036 conn(0x7effe40fa900 msgr2=0x7effe40fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:17.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 shutdown_connections 2024-04-06T14:05:17.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:17.293+0000 7effeae94700 1 -- 172.21.15.90:0/3353489036 wait complete. 2024-04-06T14:05:17.305 INFO:teuthology.run_tasks:Running task kclient... 2024-04-06T14:05:17.329 INFO:tasks.kclient:Mounting kernel clients... 2024-04-06T14:05:17.329 INFO:tasks.kclient:config is {'client.0': {}} 2024-04-06T14:05:17.330 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:17.330 DEBUG:teuthology.orchestra.run.smithi090:> ip netns list 2024-04-06T14:05:17.349 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:17.350 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link delete ceph-brx 2024-04-06T14:05:17.419 INFO:teuthology.orchestra.run.smithi090.stderr:Cannot find device "ceph-brx" 2024-04-06T14:05:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:05:17.421 INFO:tasks.kclient:client.0 config is {'mntopts': ['ms_mode=secure', 'wsync', 'context=system_u:object_r:postgresql_db_t:s0'], 'syntax': 'v1', 'mount_subvol_num': 1, 'mountpoint': '/tmp/cephfs'} 2024-04-06T14:05:17.421 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-04-06T14:05:17.421 INFO:tasks.cephfs.mount:using client_config["hostfs_mntpt"] = /tmp/cephfs 2024-04-06T14:05:17.421 INFO:tasks.cephfs.mount:hostfs_mntpt = /tmp/cephfs 2024-04-06T14:05:17.422 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph fs ls 2024-04-06T14:05:17.484 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:17.293+0000 7f923225c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T14:05:17.496 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:17.245+0000 7f721fb6f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T14:05:17.540 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:17.253+0000 7f09e93ef200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-06T14:05:17.691 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:05:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-06T14:05:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-mon[43420]: mgrmap e18: x(active, since 6m), standbys: z, y 2024-04-06T14:05:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1251859673' entity='client.admin' 2024-04-06T14:05:17.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3353489036' entity='client.admin' 2024-04-06T14:05:17.873 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-06T14:05:17.873 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-mon[46969]: mgrmap e18: x(active, since 6m), standbys: z, y 2024-04-06T14:05:17.873 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1251859673' entity='client.admin' 2024-04-06T14:05:17.874 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3353489036' entity='client.admin' 2024-04-06T14:05:17.880 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1221283268' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2024-04-06T14:05:17.881 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-mon[47549]: mgrmap e18: x(active, since 6m), standbys: z, y 2024-04-06T14:05:17.881 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1251859673' entity='client.admin' 2024-04-06T14:05:17.881 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3353489036' entity='client.admin' 2024-04-06T14:05:18.124 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:17 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:17.872+0000 7f721fb6f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:18.132 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:17.880+0000 7f09e93ef200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:18.222 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:17.958+0000 7f923225c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:18.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.132+0000 7f09e93ef200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T14:05:18.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.218+0000 7f09e93ef200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T14:05:18.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.326+0000 7f09e93ef200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:18.482 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.122+0000 7f721fb6f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T14:05:18.482 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.208+0000 7f721fb6f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T14:05:18.482 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.317+0000 7f721fb6f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:18.488 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.223+0000 7f923225c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-06T14:05:18.488 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.312+0000 7f923225c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-06T14:05:18.488 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.422+0000 7f923225c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:18.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.473+0000 7f721fb6f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T14:05:18.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.600+0000 7f721fb6f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T14:05:18.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:18.686+0000 7f721fb6f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T14:05:18.789 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.591+0000 7f923225c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T14:05:18.789 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.726+0000 7f923225c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T14:05:18.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 -- 172.21.15.90:0/2240472840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 msgr2=0x7f90d8107220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 --2- 172.21.15.90:0/2240472840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d8107220 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7f90d4009990 tx=0x7f90d4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 -- 172.21.15.90:0/2240472840 shutdown_connections 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 --2- 172.21.15.90:0/2240472840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 0x7f90d810dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 --2- 172.21.15.90:0/2240472840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d8107220 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 --2- 172.21.15.90:0/2240472840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d8101260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.855+0000 7f90e010d700 1 -- 172.21.15.90:0/2240472840 >> 172.21.15.90:0/2240472840 conn(0x7f90d8077fa0 msgr2=0x7f90d80783b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.856+0000 7f90e010d700 1 -- 172.21.15.90:0/2240472840 shutdown_connections 2024-04-06T14:05:18.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.856+0000 7f90e010d700 1 -- 172.21.15.90:0/2240472840 wait complete. 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.856+0000 7f90e010d700 1 Processor -- start 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.856+0000 7f90e010d700 1 -- start start 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 0x7f90d81976f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d819baa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d819c0b0 con 0x7f90d8106e40 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d819c220 con 0x7f90d81017a0 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f90d819c3e0 con 0x7f90d8100e00 2024-04-06T14:05:18.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90de6aa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d819baa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90de6aa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d819baa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40362/0 (socket says 172.21.15.90:40362) 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:41812/0 (socket says 172.21.15.90:41812) 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90de6aa700 1 -- 172.21.15.90:0/3444111080 learned_addr learned my addr 172.21.15.90:0/3444111080 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90dd6a8700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 0x7f90d81976f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 msgr2=0x7f90d81976f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 0x7f90d81976f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 msgr2=0x7f90d819baa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d819baa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:18.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 -- 172.21.15.90:0/3444111080 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f90d40095f0 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ddea9700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f90d40059d0 tx=0x7f90d4004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90d401d070 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f90d819c660 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.857+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f90d4022470 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.858+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f90d4017650 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.858+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90d81a3f20 con 0x7f90d8100e00 2024-04-06T14:05:18.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.858+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f90d8109360 con 0x7f90d8100e00 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.860+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 18) v1 ==== 135941+0+0 (secure 0 0 0) 0x7f90d40225e0 con 0x7f90d8100e00 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.860+0000 7f90ceffd700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 0x7f90c40a3740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.860+0000 7f90dd6a8700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 msgr2=0x7f90c40a3740 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.860+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 8211+0+0 (secure 0 0 0) 0x7f90d40cda90 con 0x7f90d8100e00 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.860+0000 7f90dd6a8700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 0x7f90c40a3740 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-06T14:05:18.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:18.864+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f90d4097160 con 0x7f90d8100e00 2024-04-06T14:05:18.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.485+0000 7f09e93ef200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-06T14:05:18.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.612+0000 7f09e93ef200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-06T14:05:18.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:18.698+0000 7f09e93ef200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T14:05:19.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.061+0000 7f90dd6a8700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 msgr2=0x7f90c40a3740 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.90:6800/1992032630 2024-04-06T14:05:19.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.061+0000 7f90dd6a8700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 0x7f90c40a3740 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-06T14:05:19.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.107+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f90d81984c0 con 0x7f90d8100e00 2024-04-06T14:05:19.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.108+0000 7f90ceffd700 1 -- 172.21.15.90:0/3444111080 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v23) v1 ==== 53+0+73 (secure 0 0 0) 0x7f90d409a780 con 0x7f90d8100e00 2024-04-06T14:05:19.107 INFO:teuthology.orchestra.run.smithi090.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data ] 2024-04-06T14:05:19.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 msgr2=0x7f90c40a3740 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:05:19.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 0x7f90c40a3740 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:19.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 msgr2=0x7f90d81971b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:19.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f90d40059d0 tx=0x7f90d4004730 comp rx=0 tx=0).stop 2024-04-06T14:05:19.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 shutdown_connections 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:6800/1992032630,v1:172.21.15.90:6801/1992032630] conn(0x7f90c40a1280 0x7f90c40a3740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f90d8106e40 0x7f90d819baa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f90d8100e00 0x7f90d81971b0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 --2- 172.21.15.90:0/3444111080 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f90d81017a0 0x7f90d81976f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 >> 172.21.15.90:0/3444111080 conn(0x7f90d8077fa0 msgr2=0x7f90d80ffcf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 shutdown_connections 2024-04-06T14:05:19.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:19.110+0000 7f90e010d700 1 -- 172.21.15.90:0/3444111080 wait complete. 2024-04-06T14:05:19.132 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:18.817+0000 7f923225c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-06T14:05:19.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:19 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3444111080' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-06T14:05:19.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:19.070+0000 7f09e93ef200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T14:05:19.439 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:19.052+0000 7f721fb6f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T14:05:19.440 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:19 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3444111080' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-06T14:05:19.463 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:19 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3444111080' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-06T14:05:19.463 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:19.206+0000 7f923225c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi090.front.sepia.ceph.com 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /tmp/cephfs 2024-04-06T14:05:19.511 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-04-06T14:05:19.512 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-04-06T14:05:19.512 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-04-06T14:05:19.512 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/tmp/cephfs' 2024-04-06T14:05:19.512 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:19.512 DEBUG:teuthology.orchestra.run.smithi090:> ip addr 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout: inet 127.0.0.1/8 scope host lo 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout: valid_lft forever preferred_lft forever 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout: inet6 ::1/128 scope host 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout: valid_lft forever preferred_lft forever 2024-04-06T14:05:19.570 INFO:teuthology.orchestra.run.smithi090.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: link/ether 0c:c4:7a:d8:a1:a8 brd ff:ff:ff:ff:ff:ff 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: altname enp7s0f0 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: link/ether 0c:c4:7a:d8:a1:a9 brd ff:ff:ff:ff:ff:ff 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: altname enp7s0f1 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: link/ether 0c:c4:7a:8f:d7:66 brd ff:ff:ff:ff:ff:ff 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: link/ether 0c:c4:7a:8f:d7:67 brd ff:ff:ff:ff:ff:ff 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: inet 172.21.15.90/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: valid_lft 41859sec preferred_lft 41859sec 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: inet6 fe80::aac0:3b83:4c6e:ef49/64 scope link noprefixroute 2024-04-06T14:05:19.571 INFO:teuthology.orchestra.run.smithi090.stdout: valid_lft forever preferred_lft forever 2024-04-06T14:05:19.573 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-04-06T14:05:19.573 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> (cd / && exec bash -c ' 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> set -e 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link add name ceph-brx type bridge 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip addr flush dev ceph-brx 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link set ceph-brx up 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-04-06T14:05:19.574 DEBUG:teuthology.orchestra.run.smithi090:> ') 2024-04-06T14:05:19.719 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:19.611+0000 7f923225c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T14:05:19.727 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:19.438+0000 7f721fb6f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T14:05:19.728 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:19.634+0000 7f721fb6f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T14:05:19.747 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:19.747 DEBUG:teuthology.orchestra.run.smithi090:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-04-06T14:05:19.754 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:19.462+0000 7f09e93ef200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-06T14:05:19.754 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:19.659+0000 7f09e93ef200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T14:05:19.779 INFO:teuthology.orchestra.run.smithi090.stdout:1 2024-04-06T14:05:19.780 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:19.781 DEBUG:teuthology.orchestra.run.smithi090:> route 2024-04-06T14:05:19.850 INFO:teuthology.orchestra.run.smithi090.stdout:Kernel IP routing table 2024-04-06T14:05:19.850 INFO:teuthology.orchestra.run.smithi090.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-04-06T14:05:19.850 INFO:teuthology.orchestra.run.smithi090.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-04-06T14:05:19.850 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-04-06T14:05:19.850 INFO:teuthology.orchestra.run.smithi090.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-04-06T14:05:19.851 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> (cd / && exec bash -c ' 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> set -e 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-04-06T14:05:19.851 DEBUG:teuthology.orchestra.run.smithi090:> ') 2024-04-06T14:05:19.993 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:19.833+0000 7f923225c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-06T14:05:19.994 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:19.934+0000 7f923225c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T14:05:19.996 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:19.726+0000 7f721fb6f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T14:05:20.065 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.066 DEBUG:teuthology.orchestra.run.smithi090:> ip netns list 2024-04-06T14:05:20.084 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.084 DEBUG:teuthology.orchestra.run.smithi090:> ip netns list-id 2024-04-06T14:05:20.142 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.142 DEBUG:teuthology.orchestra.run.smithi090:> (cd / && exec bash -c ' 2024-04-06T14:05:20.142 DEBUG:teuthology.orchestra.run.smithi090:> set -e 2024-04-06T14:05:20.142 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns add ceph-ns--tmp-cephfs 2024-04-06T14:05:20.142 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns set ceph-ns--tmp-cephfs 0 2024-04-06T14:05:20.142 DEBUG:teuthology.orchestra.run.smithi090:> ') 2024-04-06T14:05:20.143 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:19.753+0000 7f09e93ef200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-06T14:05:20.240 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--tmp-cephfs' with 192.168.0.1/16 2024-04-06T14:05:20.241 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> (cd / && exec bash -c ' 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> set -e 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link add veth0 netns ceph-ns--tmp-cephfs type veth peer name brx.0 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns exec ceph-ns--tmp-cephfs ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns exec ceph-ns--tmp-cephfs ip link set veth0 up 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns exec ceph-ns--tmp-cephfs ip link set lo up 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns exec ceph-ns--tmp-cephfs ip route add default via 192.168.255.254 2024-04-06T14:05:20.241 DEBUG:teuthology.orchestra.run.smithi090:> ') 2024-04-06T14:05:20.468 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.468 DEBUG:teuthology.orchestra.run.smithi090:> (cd / && exec bash -c ' 2024-04-06T14:05:20.468 DEBUG:teuthology.orchestra.run.smithi090:> set -e 2024-04-06T14:05:20.468 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link set brx.0 up 2024-04-06T14:05:20.468 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link set dev brx.0 master ceph-brx 2024-04-06T14:05:20.469 DEBUG:teuthology.orchestra.run.smithi090:> ') 2024-04-06T14:05:20.569 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-06T14:05:20.569 DEBUG:teuthology.orchestra.run.smithi090:> mkdir -p -v /tmp/cephfs 2024-04-06T14:05:20.586 INFO:teuthology.orchestra.run.smithi090.stdout:mkdir: created directory '/tmp/cephfs' 2024-04-06T14:05:20.586 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-06T14:05:20.586 DEBUG:teuthology.orchestra.run.smithi090:> chmod 0000 /tmp/cephfs 2024-04-06T14:05:20.643 WARNING:tasks.cephfs.mount:You cannot specify both: cephfs_mntpt and mount_subvol_num 2024-04-06T14:05:20.643 INFO:tasks.cephfs.mount:Mounting subvol 1 for now 2024-04-06T14:05:20.643 INFO:tasks.cephfs.kernel_mount:mounting using device: :/volumes/qa/sv_1/b6ecb66e-1758-4685-8e5c-896930794149 2024-04-06T14:05:20.644 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:20.644 DEBUG:teuthology.orchestra.run.smithi090:> sudo nsenter --net=/var/run/netns/ceph-ns--tmp-cephfs adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/volumes/qa/sv_1/b6ecb66e-1758-4685-8e5c-896930794149 /tmp/cephfs -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context=system_u:object_r:postgresql_db_t:s0 2024-04-06T14:05:20.730 INFO:teuthology.orchestra.run.smithi090.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context="system_u:object_r:postgresql_db_t:s0" 2024-04-06T14:05:20.730 INFO:teuthology.orchestra.run.smithi090.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context="system_u:object_r:postgresql_db_t:s0"". 2024-04-06T14:05:20.730 INFO:teuthology.orchestra.run.smithi090.stdout:invalid new device string format 2024-04-06T14:05:20.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:20 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:20.484+0000 7f721fb6f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T14:05:20.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:20 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:20.579+0000 7f721fb6f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T14:05:20.748 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:20 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:20.727+0000 7f721fb6f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T14:05:20.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:20.761+0000 7f923225c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T14:05:20.892 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:20.519+0000 7f09e93ef200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-06T14:05:20.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:20.617+0000 7f09e93ef200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T14:05:20.893 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:20.766+0000 7f09e93ef200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T14:05:21.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:20.863+0000 7f923225c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-06T14:05:21.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:21.011+0000 7f923225c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-06T14:05:21.730 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:21.350+0000 7f721fb6f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T14:05:21.730 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:21.535+0000 7f721fb6f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T14:05:21.780 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:21.399+0000 7f09e93ef200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T14:05:21.780 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:21.583+0000 7f09e93ef200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T14:05:21.997 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:21.730+0000 7f721fb6f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T14:05:22.023 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:21.639+0000 7f923225c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-06T14:05:22.023 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:21.824+0000 7f923225c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-06T14:05:22.023 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:22.023+0000 7f923225c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T14:05:22.142 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:21.779+0000 7f09e93ef200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-06T14:05:22.940 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:22 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:22.656+0000 7f721fb6f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T14:05:23.000 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:22.715+0000 7f09e93ef200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T14:05:23.218 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:22 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:22.940+0000 7f721fb6f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:23.218 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:23.028+0000 7f721fb6f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:23.247 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:22.954+0000 7f923225c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-06T14:05:23.281 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:22.999+0000 7f09e93ef200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:23.281 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:23.088+0000 7f09e93ef200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:23.496 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y[48453]: 2024-04-06T14:05:23.218+0000 7f721fb6f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T14:05:23.503 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.246+0000 7f923225c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-06T14:05:23.503 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.338+0000 7f923225c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context="system_u:object_r:postgresql_db_t:s0" 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context="system_u:object_r:postgresql_db_t:s0"". 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:invalid new device string format 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:mount.ceph: resolved to: "172.21.15.90:3300,172.21.15.105:3300,172.21.15.134:3300" 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.90:3300,172.21.15.105:3300,172.21.15.134:3300:/volumes/qa/sv_1/b6ecb66e-1758-4685-8e5c-896930794149 2024-04-06T14:05:23.584 INFO:teuthology.orchestra.run.smithi090.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=secure,wsync,context="system_u:object_r:postgresql_db_t:s0",key=0,fsid=949dca1a-f41d-11ee-b649-cb9ed24678a4" will pass to kernel 2024-04-06T14:05:23.587 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-04-06T14:05:23.587 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:23.587 DEBUG:teuthology.orchestra.run.smithi090:> sudo chmod 1777 /tmp/cephfs 2024-04-06T14:05:23.636 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:23.637 DEBUG:teuthology.orchestra.run.smithi090:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-06T14:05:23.642 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z[49410]: 2024-04-06T14:05:23.280+0000 7f09e93ef200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: Standby manager daemon y restarted 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: Standby manager daemon y started 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: Standby manager daemon z restarted 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: Standby manager daemon z started 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:05:23.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:23 smithi134 ceph-mon[47549]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.670 INFO:teuthology.orchestra.run.smithi090.stdout:client24916 2024-04-06T14:05:23.671 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T14:05:23.671 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-06T14:05:23.731 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:23.731 DEBUG:teuthology.orchestra.run.smithi090:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-06T14:05:23.788 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.537+0000 7f923225c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-06T14:05:23.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: Standby manager daemon y restarted 2024-04-06T14:05:23.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: Standby manager daemon y started 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: Standby manager daemon z restarted 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: Standby manager daemon z started 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:05:23.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-mon[43420]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.810 INFO:teuthology.orchestra.run.smithi090.stdout:client24916 2024-04-06T14:05:23.811 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:23.811 DEBUG:teuthology.orchestra.run.smithi090:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-06T14:05:23.884 INFO:teuthology.orchestra.run.smithi090.stdout:client24916 2024-04-06T14:05:23.884 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:23.884 DEBUG:teuthology.orchestra.run.smithi090:> (cd /tmp/cephfs && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4.client24916/status') 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: Standby manager daemon y restarted 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: Standby manager daemon y started 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.105:0/2199602607' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: Standby manager daemon z restarted 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: Standby manager daemon z started 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2024-04-06T14:05:23.901 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-06T14:05:23.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2024-04-06T14:05:23.902 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:23 smithi105 ceph-mon[46969]: from='mgr.? 172.21.15.134:0/2232564586' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-06T14:05:23.969 INFO:teuthology.orchestra.run.smithi090.stdout:instance: client.24916 (3)172.21.15.90:0/3220963813 2024-04-06T14:05:23.969 INFO:teuthology.orchestra.run.smithi090.stdout:blocklisted: false 2024-04-06T14:05:23.970 INFO:teuthology.orchestra.run.smithi090.stderr:0+1 records in 2024-04-06T14:05:23.970 INFO:teuthology.orchestra.run.smithi090.stderr:0+1 records out 2024-04-06T14:05:23.970 INFO:teuthology.orchestra.run.smithi090.stderr:71 bytes copied, 2.6229e-05 s, 2.7 MB/s 2024-04-06T14:05:23.971 INFO:teuthology.run_tasks:Running task exec... 2024-04-06T14:05:23.981 INFO:teuthology.task.exec:Executing custom commands... 2024-04-06T14:05:23.981 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:05:23.981 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs set cephfs balance_automate false' 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 -- 172.21.15.90:0/862063903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 msgr2=0x7f68b01091f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 --2- 172.21.15.90:0/862063903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b01091f0 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f68a800b0a0 tx=0x7f68a800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 -- 172.21.15.90:0/862063903 shutdown_connections 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 --2- 172.21.15.90:0/862063903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b01091f0 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 --2- 172.21.15.90:0/862063903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b00ff3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 --2- 172.21.15.90:0/862063903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0102400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 -- 172.21.15.90:0/862063903 >> 172.21.15.90:0/862063903 conn(0x7f68b00fa900 msgr2=0x7f68b00fcd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:24.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 -- 172.21.15.90:0/862063903 shutdown_connections 2024-04-06T14:05:24.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 -- 172.21.15.90:0/862063903 wait complete. 2024-04-06T14:05:24.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.114+0000 7f68b62ab700 1 Processor -- start 2024-04-06T14:05:24.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 -- start start 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b019e320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68b019e8f0 con 0x7f68b0102940 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68b019ea30 con 0x7f68b0101fa0 2024-04-06T14:05:24.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b62ab700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68b019ebc0 con 0x7f68b00fefd0 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68affff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68af7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68b4848700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b019e320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68affff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:56602/0 (socket says 172.21.15.90:56602) 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68af7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39994/0 (socket says 172.21.15.90:39994) 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.115+0000 7f68affff700 1 -- 172.21.15.90:0/1337817139 learned_addr learned my addr 172.21.15.90:0/1337817139 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 -- 172.21.15.90:0/1337817139 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 msgr2=0x7f68b0197300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 -- 172.21.15.90:0/1337817139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 msgr2=0x7f68b019e320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b019e320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 -- 172.21.15.90:0/1337817139 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68a0009520 con 0x7f68b00fefd0 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68af7fe700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:24.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68affff700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f68a000ea50 tx=0x7f68a000ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68a000cbd0 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68a8009d00 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f68a000cd30 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f68a0018680 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.116+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68b019f1c0 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.117+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f68b01081b0 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.118+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 20) v1 ==== 135558+0+0 (secure 0 0 0) 0x7f68a0010430 con 0x7f68b00fefd0 2024-04-06T14:05:24.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.119+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f68a0014070 con 0x7f68b00fefd0 2024-04-06T14:05:24.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.122+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f68a0096e50 con 0x7f68b00fefd0 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.831+0000 7f9190df2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.831+0000 7f9190df2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.831+0000 7f9190df2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.831+0000 7f9190df2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.831+0000 7f9190df2700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.833+0000 7f9194dfa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.833+0000 7f9194dfa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.833+0000 7f9194dfa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.833+0000 7f9194dfa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.289 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:23.833+0000 7f9194dfa700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:24.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.406+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"} v 0) v1 -- 0x7f68b00673b0 con 0x7f68b00fefd0 2024-04-06T14:05:24.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.559+0000 7f68ad7fa700 1 -- 172.21.15.90:0/1337817139 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]=0 v24) v1 ==== 118+0+0 (secure 0 0 0) 0x7f68a009a470 con 0x7f68b00fefd0 2024-04-06T14:05:24.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 msgr2=0x7f68b0196dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f68a000ea50 tx=0x7f68a000ed60 comp rx=0 tx=0).stop 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 shutdown_connections 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f68b0102940 0x7f68b019e320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f68b00fefd0 0x7f68b0196dc0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 --2- 172.21.15.90:0/1337817139 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f68b0101fa0 0x7f68b0197300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 >> 172.21.15.90:0/1337817139 conn(0x7f68b00fa900 msgr2=0x7f68b01052f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 shutdown_connections 2024-04-06T14:05:24.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.562+0000 7f68b62ab700 1 -- 172.21.15.90:0/1337817139 wait complete. 2024-04-06T14:05:24.584 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs subvolumegroup pin cephfs qa random 0.10' 2024-04-06T14:05:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 -- 172.21.15.90:0/1260784777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c106a00 msgr2=0x7f0d8c108e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 --2- 172.21.15.90:0/1260784777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c106a00 0x7f0d8c108e70 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f0d8400b0a0 tx=0x7f0d8400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 -- 172.21.15.90:0/1260784777 shutdown_connections 2024-04-06T14:05:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 --2- 172.21.15.90:0/1260784777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c106a00 0x7f0d8c108e70 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 --2- 172.21.15.90:0/1260784777 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c1017a0 0x7f0d8c103b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 --2- 172.21.15.90:0/1260784777 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c1064c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 -- 172.21.15.90:0/1260784777 >> 172.21.15.90:0/1260784777 conn(0x7f0d8c0fb190 msgr2=0x7f0d8c0fd5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 -- 172.21.15.90:0/1260784777 shutdown_connections 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.714+0000 7f0d92623700 1 -- 172.21.15.90:0/1260784777 wait complete. 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 Processor -- start 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 -- start start 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1017a0 0x7f0d8c1010c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 0x7f0d8c0ffc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c100430 con 0x7f0d8c1040d0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c1005a0 con 0x7f0d8c106a00 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d92623700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d8c1a0db0 con 0x7f0d8c1017a0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50092/0 (socket says 172.21.15.90:50092) 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 -- 172.21.15.90:0/2860490262 learned_addr learned my addr 172.21.15.90:0/2860490262 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d90bc0700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 0x7f0d8c0ffc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8bfff700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1017a0 0x7f0d8c1010c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 -- 172.21.15.90:0/2860490262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1017a0 msgr2=0x7f0d8c1010c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1017a0 0x7f0d8c1010c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 -- 172.21.15.90:0/2860490262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 msgr2=0x7f0d8c0ffc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 0x7f0d8c0ffc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.715+0000 7f0d8b7fe700 1 -- 172.21.15.90:0/2860490262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d84009d00 con 0x7f0d8c1040d0 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d90bc0700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 0x7f0d8c0ffc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:24.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d8b7fe700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f0d8000d950 tx=0x7f0d8000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d8000f890 con 0x7f0d8c1040d0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d80010460 con 0x7f0d8c1040d0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d8c1a0fe0 con 0x7f0d8c1040d0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d800195c0 con 0x7f0d8c1040d0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.716+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d8c1a14c0 con 0x7f0d8c1040d0 2024-04-06T14:05:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.717+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d8c0fcd10 con 0x7f0d8c1040d0 2024-04-06T14:05:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.718+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 21) v1 ==== 135685+0+0 (secure 0 0 0) 0x7f0d800108d0 con 0x7f0d8c1040d0 2024-04-06T14:05:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.719+0000 7f0d897fa700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 0x7f0d740a33e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.719+0000 7f0d8bfff700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 0x7f0d740a33e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.719+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0d800cc350 con 0x7f0d8c1040d0 2024-04-06T14:05:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.720+0000 7f0d8bfff700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 0x7f0d740a33e0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f0d7c009c60 tx=0x7f0d7c009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:24.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.723+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0d80095cc0 con 0x7f0d8c1040d0 2024-04-06T14:05:24.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: mgrmap e19: x(active, since 6m), standbys: z, y 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: Active manager daemon x restarted 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: Activating manager daemon x 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: osdmap e84: 12 total, 12 up, 12 in 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: mgrmap e20: x(active, starting, since 0.0138181s), standbys: z, y 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:05:24.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:05:24.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: Manager daemon x is now available 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T14:05:24.791 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T14:05:24.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1337817139' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:24.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-mon[43420]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: mgrmap e19: x(active, since 6m), standbys: z, y 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: Active manager daemon x restarted 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: Activating manager daemon x 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: osdmap e84: 12 total, 12 up, 12 in 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: mgrmap e20: x(active, starting, since 0.0138181s), standbys: z, y 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:05:24.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:05:24.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:05:24.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: Manager daemon x is now available 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1337817139' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:24.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:24 smithi134 ceph-mon[47549]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:24.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.969+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}) v1 -- 0x7f0d8c02cdc0 con 0x7f0d740a0f20 2024-04-06T14:05:24.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.988+0000 7f0d897fa700 1 -- 172.21.15.90:0/2860490262 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+3 (secure 0 0 0) 0x7f0d8c02cdc0 con 0x7f0d740a0f20 2024-04-06T14:05:24.989 INFO:teuthology.orchestra.run.smithi090.stdout:{} 2024-04-06T14:05:24.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 msgr2=0x7f0d740a33e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 0x7f0d740a33e0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f0d7c009c60 tx=0x7f0d7c009260 comp rx=0 tx=0).stop 2024-04-06T14:05:24.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 msgr2=0x7f0d8c0ff750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:24.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f0d8000d950 tx=0x7f0d8000dc60 comp rx=0 tx=0).stop 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 shutdown_connections 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d740a0f20 0x7f0d740a33e0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d8c106a00 0x7f0d8c0ffc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d8c1017a0 0x7f0d8c1010c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 --2- 172.21.15.90:0/2860490262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d8c1040d0 0x7f0d8c0ff750 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.991+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 >> 172.21.15.90:0/2860490262 conn(0x7f0d8c0fb190 msgr2=0x7f0d8c102060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.992+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 shutdown_connections 2024-04-06T14:05:24.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:24.992+0000 7f0d92623700 1 -- 172.21.15.90:0/2860490262 wait complete. 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: mgrmap e19: x(active, since 6m), standbys: z, y 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: Active manager daemon x restarted 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: Activating manager daemon x 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: osdmap e84: 12 total, 12 up, 12 in 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: mgrmap e20: x(active, starting, since 0.0138181s), standbys: z, y 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-06T14:05:24.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:05:24.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2024-04-06T14:05:24.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: Manager daemon x is now available 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1337817139' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:25.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:24 smithi105 ceph-mon[46969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]: dispatch 2024-04-06T14:05:25.006 INFO:teuthology.run_tasks:Running task check-counter... 2024-04-06T14:05:25.018 INFO:teuthology.run_tasks:Running task fwd_scrub... 2024-04-06T14:05:25.039 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 -- 172.21.15.90:0/3918159033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c101f00 msgr2=0x7fe59c102360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/3918159033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c101f00 0x7fe59c102360 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fe58c009990 tx=0x7fe58c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 -- 172.21.15.90:0/3918159033 shutdown_connections 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/3918159033 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c109150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/3918159033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c0fef30 0x7fe59c0ff350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/3918159033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c101f00 0x7fe59c102360 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 -- 172.21.15.90:0/3918159033 >> 172.21.15.90:0/3918159033 conn(0x7fe59c0fa880 msgr2=0x7fe59c0fcca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:25.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 -- 172.21.15.90:0/3918159033 shutdown_connections 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.189+0000 7fe5a38f2700 1 -- 172.21.15.90:0/3918159033 wait complete. 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 Processor -- start 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 -- start start 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 0x7fe59c197110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c19e130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe59c19e700 con 0x7fe59c0fef30 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe59c19e840 con 0x7fe59c101f00 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a38f2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe59c19e9d0 con 0x7fe59c1028a0 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a168e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a168e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50116/0 (socket says 172.21.15.90:50116) 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 0x7fe59c197110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a168e700 1 -- 172.21.15.90:0/1066230462 learned_addr learned my addr 172.21.15.90:0/1066230462 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a1e8f700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c19e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 -- 172.21.15.90:0/1066230462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 msgr2=0x7fe59c19e130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c19e130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 -- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 msgr2=0x7fe59c196bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 -- 172.21.15.90:0/1066230462 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe58c0095f0 con 0x7fe59c101f00 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a1e8f700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c19e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a168e700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.190+0000 7fe5a0e8d700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 0x7fe59c197110 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fe58c000c00 tx=0x7fe58c004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.191+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe58c01d070 con 0x7fe59c101f00 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.191+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe59c19ec50 con 0x7fe59c101f00 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.191+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe58c021410 con 0x7fe59c101f00 2024-04-06T14:05:25.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.191+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe58c021ea0 con 0x7fe59c101f00 2024-04-06T14:05:25.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.191+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe59c19f0d0 con 0x7fe59c101f00 2024-04-06T14:05:25.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.193+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe59c108130 con 0x7fe59c101f00 2024-04-06T14:05:25.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.193+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 21) v1 ==== 135685+0+0 (secure 0 0 0) 0x7fe58c021580 con 0x7fe59c101f00 2024-04-06T14:05:25.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.193+0000 7fe5927fc700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 0x7fe5880a33e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.194+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe58c0cd6b0 con 0x7fe59c101f00 2024-04-06T14:05:25.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.194+0000 7fe5a168e700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 0x7fe5880a33e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.194+0000 7fe5a168e700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 0x7fe5880a33e0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe598005950 tx=0x7fe598009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.198+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe58c0970b0 con 0x7fe59c101f00 2024-04-06T14:05:25.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:24.974+0000 7f918edee700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:25.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:24.974+0000 7f918edee700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:25.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:24.974+0000 7f918edee700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:25.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:24.974+0000 7f918edee700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:25.288 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:05:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:05:24.974+0000 7f918edee700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:05:25.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.413+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe59c02ccb0 con 0x7fe59c101f00 2024-04-06T14:05:25.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.415+0000 7fe5927fc700 1 -- 172.21.15.90:0/1066230462 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v84) v1 ==== 74+0+22920 (secure 0 0 0) 0x7fe58c026090 con 0x7fe59c101f00 2024-04-06T14:05:25.415 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:25.416 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":84,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:23.551387+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 msgr2=0x7fe5880a33e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 0x7fe5880a33e0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe598005950 tx=0x7fe598009260 comp rx=0 tx=0).stop 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 msgr2=0x7fe59c197110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 0x7fe59c197110 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fe58c000c00 tx=0x7fe58c004730 comp rx=0 tx=0).stop 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 shutdown_connections 2024-04-06T14:05:25.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5880a0f20 0x7fe5880a33e0 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe59c1028a0 0x7fe59c19e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe59c0fef30 0x7fe59c196bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 --2- 172.21.15.90:0/1066230462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe59c101f00 0x7fe59c197110 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 >> 172.21.15.90:0/1066230462 conn(0x7fe59c0fa880 msgr2=0x7fe59c105270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 shutdown_connections 2024-04-06T14:05:25.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.417+0000 7fe5a38f2700 1 -- 172.21.15.90:0/1066230462 wait complete. 2024-04-06T14:05:25.430 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:05:25.431 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:05:25.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 -- 172.21.15.90:0/1899480177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08100e80 msgr2=0x7fda081012e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 --2- 172.21.15.90:0/1899480177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08100e80 0x7fda081012e0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7fd9f80099c0 tx=0x7fd9f8009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 -- 172.21.15.90:0/1899480177 shutdown_connections 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 --2- 172.21.15.90:0/1899480177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08101820 0x7fda081080d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 --2- 172.21.15.90:0/1899480177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda081000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 --2- 172.21.15.90:0/1899480177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08100e80 0x7fda081012e0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 -- 172.21.15.90:0/1899480177 >> 172.21.15.90:0/1899480177 conn(0x7fda080fb200 msgr2=0x7fda080fd640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 -- 172.21.15.90:0/1899480177 shutdown_connections 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.571+0000 7fda101df700 1 -- 172.21.15.90:0/1899480177 wait complete. 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 Processor -- start 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 -- start start 2024-04-06T14:05:25.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08100e80 0x7fda08108be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 0x7fda08109120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda081098c0 con 0x7fda08101820 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda08109a30 con 0x7fda080ffc90 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda101df700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda081a0d90 con 0x7fda08100e80 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda0df7b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda0e77c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 0x7fda08109120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda0df7b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:40052/0 (socket says 172.21.15.90:40052) 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda0df7b700 1 -- 172.21.15.90:0/1313254568 learned_addr learned my addr 172.21.15.90:0/1313254568 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.572+0000 7fda0d77a700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08100e80 0x7fda08108be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 -- 172.21.15.90:0/1313254568 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08100e80 msgr2=0x7fda08108be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08100e80 0x7fda08108be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 -- 172.21.15.90:0/1313254568 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 msgr2=0x7fda0810a550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 -- 172.21.15.90:0/1313254568 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9f80095f0 con 0x7fda08101820 2024-04-06T14:05:25.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0df7b700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:25.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda0e77c700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 0x7fda08109120 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fda0000be20 tx=0x7fda00009ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda0000cb30 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fda081a0fc0 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda0000cc90 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda000125a0 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.573+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda081a14f0 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.574+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fda081070c0 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.575+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 21) v1 ==== 135685+0+0 (secure 0 0 0) 0x7fda00014440 con 0x7fda08101820 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.576+0000 7fd9feffd700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 0x7fd9f40a33e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.576+0000 7fda0df7b700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 0x7fd9f40a33e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.576+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fda00006e40 con 0x7fda08101820 2024-04-06T14:05:25.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.577+0000 7fda0df7b700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 0x7fd9f40a33e0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fda040105c0 tx=0x7fda0400b4a0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.579+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fda00095bd0 con 0x7fda08101820 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: [06/Apr/2024:14:05:24] ENGINE Bus STARTING 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: [06/Apr/2024:14:05:24] ENGINE Serving on http://172.21.15.90:8765 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: [06/Apr/2024:14:05:24] ENGINE Serving on https://172.21.15.90:7150 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: [06/Apr/2024:14:05:24] ENGINE Bus STARTED 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: mgrmap e21: x(active, since 1.02093s), standbys: z, y 2024-04-06T14:05:25.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:25 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1066230462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:25.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.794+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fda0810c650 con 0x7fda08101820 2024-04-06T14:05:25.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.794+0000 7fd9feffd700 1 -- 172.21.15.90:0/1313254568 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v84) v1 ==== 93+0+10 (secure 0 0 0) 0x7fda00023020 con 0x7fda08101820 2024-04-06T14:05:25.794 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 msgr2=0x7fd9f40a33e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 0x7fd9f40a33e0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fda040105c0 tx=0x7fda0400b4a0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 msgr2=0x7fda08109120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 0x7fda08109120 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7fda0000be20 tx=0x7fda00009ed0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 shutdown_connections 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd9f40a0f20 0x7fd9f40a33e0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fda08101820 0x7fda08109120 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fda080ffc90 0x7fda0810a550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 --2- 172.21.15.90:0/1313254568 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fda08100e80 0x7fda08108be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 >> 172.21.15.90:0/1313254568 conn(0x7fda080fb200 msgr2=0x7fda08104200 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:25.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.796+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 shutdown_connections 2024-04-06T14:05:25.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.797+0000 7fda101df700 1 -- 172.21.15.90:0/1313254568 wait complete. 2024-04-06T14:05:25.811 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: [06/Apr/2024:14:05:24] ENGINE Bus STARTING 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: [06/Apr/2024:14:05:24] ENGINE Serving on http://172.21.15.90:8765 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: [06/Apr/2024:14:05:24] ENGINE Serving on https://172.21.15.90:7150 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: [06/Apr/2024:14:05:24] ENGINE Bus STARTED 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: mgrmap e21: x(active, since 1.02093s), standbys: z, y 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:25 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1066230462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 -- 172.21.15.90:0/2604160271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 msgr2=0x7efc841023d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/2604160271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc841023d0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7efc740099c0 tx=0x7efc74009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 -- 172.21.15.90:0/2604160271 shutdown_connections 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/2604160271 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc841091c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/2604160271 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 0x7efc841011a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/2604160271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc841023d0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 -- 172.21.15.90:0/2604160271 >> 172.21.15.90:0/2604160271 conn(0x7efc840fc2f0 msgr2=0x7efc840fe730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 -- 172.21.15.90:0/2604160271 shutdown_connections 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.943+0000 7efc8b5e3700 1 -- 172.21.15.90:0/2604160271 wait complete. 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 Processor -- start 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 -- start start 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 0x7efc84196cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc8419e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc8419e7c0 con 0x7efc84101f70 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc8419e930 con 0x7efc84100d80 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8b5e3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc8419eaf0 con 0x7efc84102910 2024-04-06T14:05:25.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc88b7e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc88b7e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50184/0 (socket says 172.21.15.90:50184) 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc88b7e700 1 -- 172.21.15.90:0/3423172167 learned_addr learned my addr 172.21.15.90:0/3423172167 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc8937f700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 0x7efc84196cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.944+0000 7efc89b80700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc8419e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 -- 172.21.15.90:0/3423172167 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 msgr2=0x7efc8419e1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc8419e1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 -- 172.21.15.90:0/3423172167 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 msgr2=0x7efc84196cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 0x7efc84196cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 -- 172.21.15.90:0/3423172167 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc740095f0 con 0x7efc84101f70 2024-04-06T14:05:25.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc89b80700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc8419e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc88b7e700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7efc74009fd0 tx=0x7efc74005f10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc7401d070 con 0x7efc84101f70 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc8419ed70 con 0x7efc84101f70 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc740059a0 con 0x7efc84101f70 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc740173f0 con 0x7efc84101f70 2024-04-06T14:05:25.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.945+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc8419f1d0 con 0x7efc84101f70 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.946+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc84108200 con 0x7efc84101f70 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.947+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 21) v1 ==== 135685+0+0 (secure 0 0 0) 0x7efc740226e0 con 0x7efc84101f70 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.948+0000 7efc7a7fc700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 0x7efc700a33e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.948+0000 7efc8937f700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 0x7efc700a33e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.948+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7efc740cd8b0 con 0x7efc84101f70 2024-04-06T14:05:25.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.948+0000 7efc8937f700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 0x7efc700a33e0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7efc84198150 tx=0x7efc80007400 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:25.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:25.951+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efc740972b0 con 0x7efc84101f70 2024-04-06T14:05:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: [06/Apr/2024:14:05:24] ENGINE Bus STARTING 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: [06/Apr/2024:14:05:24] ENGINE Serving on http://172.21.15.90:8765 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: [06/Apr/2024:14:05:24] ENGINE Serving on https://172.21.15.90:7150 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: [06/Apr/2024:14:05:24] ENGINE Bus STARTED 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "balance_automate", "val": "false"}]': finished 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: fsmap cephfs:3 {0=b=up:active,1=i=up:active,2=h=up:active} 3 up:standby-replay 6 up:standby 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: mgrmap e21: x(active, since 1.02093s), standbys: z, y 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: pgmap v3: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:25 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1066230462' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:26.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.163+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7efc8410c590 con 0x7efc84101f70 2024-04-06T14:05:26.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.163+0000 7efc7a7fc700 1 -- 172.21.15.90:0/3423172167 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v84) v1 ==== 104+0+11 (secure 0 0 0) 0x7efc7409a8d0 con 0x7efc84101f70 2024-04-06T14:05:26.164 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 msgr2=0x7efc700a33e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 0x7efc700a33e0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7efc84198150 tx=0x7efc80007400 comp rx=0 tx=0).stop 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 msgr2=0x7efc84197210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7efc74009fd0 tx=0x7efc74005f10 comp rx=0 tx=0).stop 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 shutdown_connections 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc700a0f20 0x7efc700a33e0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc84102910 0x7efc8419e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc84100d80 0x7efc84196cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 --2- 172.21.15.90:0/3423172167 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc84101f70 0x7efc84197210 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 >> 172.21.15.90:0/3423172167 conn(0x7efc840fc2f0 msgr2=0x7efc84105340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:26.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 shutdown_connections 2024-04-06T14:05:26.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.165+0000 7efc8b5e3700 1 -- 172.21.15.90:0/3423172167 wait complete. 2024-04-06T14:05:26.179 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:05:26.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 -- 172.21.15.90:0/238323883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68106a50 msgr2=0x7f0c68108ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/238323883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68106a50 0x7f0c68108ec0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f0c6000b0a0 tx=0x7f0c6000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 -- 172.21.15.90:0/238323883 shutdown_connections 2024-04-06T14:05:26.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/238323883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68106a50 0x7f0c68108ec0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/238323883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c681017f0 0x7f0c68103be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/238323883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c68104120 0x7f0c68106510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.314+0000 7f0c6e13d700 1 -- 172.21.15.90:0/238323883 >> 172.21.15.90:0/238323883 conn(0x7f0c680fb200 msgr2=0x7f0c680fd620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:26.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- 172.21.15.90:0/238323883 shutdown_connections 2024-04-06T14:05:26.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- 172.21.15.90:0/238323883 wait complete. 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 Processor -- start 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- start start 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 0x7f0c68196da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c68106a50 0x7f0c6819e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c6819e890 con 0x7f0c68104120 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c6819ea00 con 0x7f0c681017f0 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.315+0000 7f0c6e13d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c6819ebc0 con 0x7f0c68106a50 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50204/0 (socket says 172.21.15.90:50204) 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c677fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 0x7f0c68196da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 -- 172.21.15.90:0/3961963849 learned_addr learned my addr 172.21.15.90:0/3961963849 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c67fff700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c68106a50 0x7f0c6819e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 -- 172.21.15.90:0/3961963849 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c68106a50 msgr2=0x7f0c6819e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c68106a50 0x7f0c6819e2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 -- 172.21.15.90:0/3961963849 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 msgr2=0x7f0c68196da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 0x7f0c68196da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 -- 172.21.15.90:0/3961963849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c50009520 con 0x7f0c68104120 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c677fe700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 0x7f0c68196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c66ffd700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f0c5000eb30 tx=0x7f0c5000ee40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c50009880 con 0x7f0c68104120 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c500044e0 con 0x7f0c68104120 2024-04-06T14:05:26.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c60009d00 con 0x7f0c68104120 2024-04-06T14:05:26.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.316+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c50003d80 con 0x7f0c68104120 2024-04-06T14:05:26.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.317+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c6819f1c0 con 0x7f0c68104120 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.317+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c680fcd80 con 0x7f0c68104120 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.320+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 21) v1 ==== 135685+0+0 (secure 0 0 0) 0x7f0c50003fe0 con 0x7f0c68104120 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.320+0000 7f0c64ff9700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 0x7f0c540a3460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.320+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0c50014070 con 0x7f0c68104120 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.320+0000 7f0c677fe700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 0x7f0c540a3460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.321+0000 7f0c677fe700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 0x7f0c540a3460 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f0c68198220 tx=0x7f0c58009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:26.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.325+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0c500965b0 con 0x7f0c68104120 2024-04-06T14:05:26.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.533+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f0c6819ed80 con 0x7f0c68104120 2024-04-06T14:05:26.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.533+0000 7f0c64ff9700 1 -- 172.21.15.90:0/3961963849 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v84) v1 ==== 100+0+11 (secure 0 0 0) 0x7f0c50004290 con 0x7f0c68104120 2024-04-06T14:05:26.534 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:05:26.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.535+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 msgr2=0x7f0c540a3460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.535+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 0x7f0c540a3460 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f0c68198220 tx=0x7f0c58009260 comp rx=0 tx=0).stop 2024-04-06T14:05:26.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.535+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 msgr2=0x7f0c681972e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.535+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7f0c5000eb30 tx=0x7f0c5000ee40 comp rx=0 tx=0).stop 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 shutdown_connections 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c540a0fa0 0x7f0c540a3460 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c68106a50 0x7f0c6819e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c681017f0 0x7f0c68196da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 --2- 172.21.15.90:0/3961963849 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c68104120 0x7f0c681972e0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 >> 172.21.15.90:0/3961963849 conn(0x7f0c680fb200 msgr2=0x7f0c68102290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 shutdown_connections 2024-04-06T14:05:26.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.536+0000 7f0c6e13d700 1 -- 172.21.15.90:0/3961963849 wait complete. 2024-04-06T14:05:26.549 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:26.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 -- 172.21.15.90:0/287204674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c1029a0 msgr2=0x7fad3c109250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 --2- 172.21.15.90:0/287204674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c1029a0 0x7fad3c109250 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7fad3400b0a0 tx=0x7fad3400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 -- 172.21.15.90:0/287204674 shutdown_connections 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 --2- 172.21.15.90:0/287204674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c1029a0 0x7fad3c109250 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 --2- 172.21.15.90:0/287204674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c0fef90 0x7fad3c0ff3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 --2- 172.21.15.90:0/287204674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c102000 0x7fad3c102460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 -- 172.21.15.90:0/287204674 >> 172.21.15.90:0/287204674 conn(0x7fad3c0fa8c0 msgr2=0x7fad3c0fcce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.690+0000 7fad42623700 1 -- 172.21.15.90:0/287204674 shutdown_connections 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 -- 172.21.15.90:0/287204674 wait complete. 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 Processor -- start 2024-04-06T14:05:26.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 -- start start 2024-04-06T14:05:26.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 0x7fad3c196cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c1029a0 0x7fad3c19e240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad3c19e810 con 0x7fad3c102000 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad3c19e980 con 0x7fad3c0fef90 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.691+0000 7fad42623700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fad3c19eb40 con 0x7fad3c1029a0 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50214/0 (socket says 172.21.15.90:50214) 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 -- 172.21.15.90:0/1407063443 learned_addr learned my addr 172.21.15.90:0/1407063443 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3bfff700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 0x7fad3c196cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad40bc0700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c1029a0 0x7fad3c19e240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 -- 172.21.15.90:0/1407063443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c1029a0 msgr2=0x7fad3c19e240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c1029a0 0x7fad3c19e240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 -- 172.21.15.90:0/1407063443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 msgr2=0x7fad3c196cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 0x7fad3c196cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 -- 172.21.15.90:0/1407063443 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fad34009d00 con 0x7fad3c102000 2024-04-06T14:05:26.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3bfff700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 0x7fad3c196cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad3b7fe700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fad3000e9a0 tx=0x7fad3000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad30009790 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fad30010430 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fad3c19ee20 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fad30018680 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.692+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fad3c19f2a0 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.693+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fad3c108230 con 0x7fad3c102000 2024-04-06T14:05:26.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.694+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 22) v1 ==== 136044+0+0 (secure 0 0 0) 0x7fad300105a0 con 0x7fad3c102000 2024-04-06T14:05:26.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.695+0000 7fad397fa700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 0x7fad240a37e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:26.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.695+0000 7fad3bfff700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 0x7fad240a37e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:26.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.695+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fad30014070 con 0x7fad3c102000 2024-04-06T14:05:26.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.696+0000 7fad3bfff700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 0x7fad240a37e0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fad3c0fc4b0 tx=0x7fad2c00b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:26.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.700+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fad30052ab0 con 0x7fad3c102000 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='client.15342 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1313254568' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:26.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3423172167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:26 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3961963849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.917+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fad3c067e70 con 0x7fad3c102000 2024-04-06T14:05:26.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.919+0000 7fad397fa700 1 -- 172.21.15.90:0/1407063443 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v84) v1 ==== 74+0+22920 (secure 0 0 0) 0x7fad30096320 con 0x7fad3c102000 2024-04-06T14:05:26.920 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:26.920 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":84,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:23.551387+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 msgr2=0x7fad240a37e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 0x7fad240a37e0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fad3c0fc4b0 tx=0x7fad2c00b410 comp rx=0 tx=0).stop 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 msgr2=0x7fad3c197230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7fad3000e9a0 tx=0x7fad3000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 shutdown_connections 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fad240a1320 0x7fad240a37e0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fad3c1029a0 0x7fad3c19e240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad3c0fef90 0x7fad3c196cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 --2- 172.21.15.90:0/1407063443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fad3c102000 0x7fad3c197230 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:26.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 >> 172.21.15.90:0/1407063443 conn(0x7fad3c0fa8c0 msgr2=0x7fad3c105370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:26.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 shutdown_connections 2024-04-06T14:05:26.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:26.921+0000 7fad42623700 1 -- 172.21.15.90:0/1407063443 wait complete. 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='client.15342 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1313254568' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3423172167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.935 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:26 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3961963849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.937 INFO:tasks.fwd_scrub.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:05:26.937 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='client.15342 -' entity='client.admin' cmd=[{"prefix": "fs subvolumegroup pin", "vol_name": "cephfs", "group_name": "qa", "pin_type": "random", "pin_setting": "0.10", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: pgmap v4: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1313254568' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:26.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3423172167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:26.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:26 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3961963849' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 -- 172.21.15.90:0/2706622690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01060a0 msgr2=0x7f4ce010dc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/2706622690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01060a0 0x7f4ce010dc20 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f4cd800b0d0 tx=0x7f4cd800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 -- 172.21.15.90:0/2706622690 shutdown_connections 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/2706622690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01060a0 0x7f4ce010dc20 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/2706622690 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 0x7f4ce00684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/2706622690 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce0068aa0 0x7f4ce0105970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 -- 172.21.15.90:0/2706622690 >> 172.21.15.90:0/2706622690 conn(0x7f4ce0076d50 msgr2=0x7f4ce0077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 -- 172.21.15.90:0/2706622690 shutdown_connections 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.067+0000 7f4ce7df6700 1 -- 172.21.15.90:0/2706622690 wait complete. 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 Processor -- start 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 -- start start 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 0x7f4ce00ffc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce01060a0 0x7f4ce01001b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce01a5a90 con 0x7f4ce01006f0 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce01a5c00 con 0x7f4ce01060a0 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce7df6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ce01a5dc0 con 0x7f4ce00680f0 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50220/0 (socket says 172.21.15.90:50220) 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 -- 172.21.15.90:0/435637717 learned_addr learned my addr 172.21.15.90:0/435637717 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:27.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce5391700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce01060a0 0x7f4ce01001b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce5b92700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 0x7f4ce00ffc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 -- 172.21.15.90:0/435637717 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 msgr2=0x7f4ce00ffc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 0x7f4ce00ffc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 -- 172.21.15.90:0/435637717 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce01060a0 msgr2=0x7f4ce01001b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce01060a0 0x7f4ce01001b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 -- 172.21.15.90:0/435637717 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cd0009520 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.068+0000 7f4ce6393700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f4cd800bcd0 tx=0x7f4cd800bd00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.069+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cd800e040 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.069+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4cd8005070 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.069+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cd8009d00 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.069+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cd801b6a0 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.069+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ce01a6330 con 0x7f4ce01006f0 2024-04-06T14:05:27.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.070+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ce01092e0 con 0x7f4ce01006f0 2024-04-06T14:05:27.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.071+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 22) v1 ==== 136044+0+0 (secure 0 0 0) 0x7f4cd80051e0 con 0x7f4ce01006f0 2024-04-06T14:05:27.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.071+0000 7f4cd6ffd700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 0x7f4ccc0a3860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.071+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4cd80cdaf0 con 0x7f4ce01006f0 2024-04-06T14:05:27.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.071+0000 7f4ce5b92700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 0x7f4ccc0a3860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.072+0000 7f4ce5b92700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 0x7f4ccc0a3860 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc009c60 tx=0x7f4cdc009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.075+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4cd8096ef0 con 0x7f4ce01006f0 2024-04-06T14:05:27.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.287+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f4ce0102450 con 0x7f4ce01006f0 2024-04-06T14:05:27.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.287+0000 7f4cd6ffd700 1 -- 172.21.15.90:0/435637717 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v84) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4cd809a510 con 0x7f4ce01006f0 2024-04-06T14:05:27.288 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:05:27.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 msgr2=0x7f4ccc0a3860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 0x7f4ccc0a3860 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f4cdc009c60 tx=0x7f4cdc009260 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 msgr2=0x7f4ce01a54c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7f4cd800bcd0 tx=0x7f4cd800bd00 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 shutdown_connections 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4ccc0a13a0 0x7f4ccc0a3860 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ce01006f0 0x7f4ce01a54c0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ce00680f0 0x7f4ce00ffc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.289+0000 7f4ce7df6700 1 --2- 172.21.15.90:0/435637717 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ce01060a0 0x7f4ce01001b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.290+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 >> 172.21.15.90:0/435637717 conn(0x7f4ce0076d50 msgr2=0x7f4ce00fde90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.290+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 shutdown_connections 2024-04-06T14:05:27.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.290+0000 7f4ce7df6700 1 -- 172.21.15.90:0/435637717 wait complete. 2024-04-06T14:05:27.303 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 -- 172.21.15.90:0/2246619488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c104bf0 msgr2=0x7fa09c104fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 --2- 172.21.15.90:0/2246619488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c104bf0 0x7fa09c104fd0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7fa090009990 tx=0x7fa090009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 -- 172.21.15.90:0/2246619488 shutdown_connections 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 --2- 172.21.15.90:0/2246619488 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c100d50 0x7fa09c10ca60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 --2- 172.21.15.90:0/2246619488 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c104bf0 0x7fa09c104fd0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 --2- 172.21.15.90:0/2246619488 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c100810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 -- 172.21.15.90:0/2246619488 >> 172.21.15.90:0/2246619488 conn(0x7fa09c0761b0 msgr2=0x7fa09c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 -- 172.21.15.90:0/2246619488 shutdown_connections 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.429+0000 7fa0a11af700 1 -- 172.21.15.90:0/2246619488 wait complete. 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 Processor -- start 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 -- start start 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c104bf0 0x7fa09c19bb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa09c19c1b0 con 0x7fa09c100d50 2024-04-06T14:05:27.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa09c19c320 con 0x7fa09c100390 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa0a11af700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa09c19c4e0 con 0x7fa09c104bf0 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:40132/0 (socket says 172.21.15.90:40132) 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50236/0 (socket says 172.21.15.90:50236) 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09ad9d700 1 -- 172.21.15.90:0/3320543710 learned_addr learned my addr 172.21.15.90:0/3320543710 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.430+0000 7fa09b59e700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c104bf0 0x7fa09c19bb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 -- 172.21.15.90:0/3320543710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c104bf0 msgr2=0x7fa09c19bb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c104bf0 0x7fa09c19bb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 -- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 msgr2=0x7fa09c1977d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 -- 172.21.15.90:0/3320543710 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0900095f0 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09a59c700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa09ad9d700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fa090005910 tx=0x7fa09000f690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa09001d070 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa090003680 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa09c19c760 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0900173f0 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.431+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa09c1a4000 con 0x7fa09c100390 2024-04-06T14:05:27.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.432+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa09c1013a0 con 0x7fa09c100390 2024-04-06T14:05:27.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.433+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 22) v1 ==== 136044+0+0 (secure 0 0 0) 0x7fa09000fad0 con 0x7fa09c100390 2024-04-06T14:05:27.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.434+0000 7fa083fff700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 0x7fa0840a3860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.434+0000 7fa09a59c700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 0x7fa0840a3860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.434+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fa0900cdd60 con 0x7fa09c100390 2024-04-06T14:05:27.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.434+0000 7fa09a59c700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 0x7fa0840a3860 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fa09c1987e0 tx=0x7fa08c00b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.437+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fa09005dab0 con 0x7fa09c100390 2024-04-06T14:05:27.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.636+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fa09c198590 con 0x7fa09c100390 2024-04-06T14:05:27.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.637+0000 7fa083fff700 1 -- 172.21.15.90:0/3320543710 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v84) v1 ==== 104+0+11 (secure 0 0 0) 0x7fa0900970a0 con 0x7fa09c100390 2024-04-06T14:05:27.637 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 msgr2=0x7fa0840a3860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 0x7fa0840a3860 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fa09c1987e0 tx=0x7fa08c00b410 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 msgr2=0x7fa09c197290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fa090005910 tx=0x7fa09000f690 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 shutdown_connections 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa0840a13a0 0x7fa0840a3860 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa09c104bf0 0x7fa09c19bb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa09c100390 0x7fa09c197290 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 --2- 172.21.15.90:0/3320543710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa09c100d50 0x7fa09c1977d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 >> 172.21.15.90:0/3320543710 conn(0x7fa09c0761b0 msgr2=0x7fa09c0ff520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 shutdown_connections 2024-04-06T14:05:27.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.639+0000 7fa0a11af700 1 -- 172.21.15.90:0/3320543710 wait complete. 2024-04-06T14:05:27.652 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 -- 172.21.15.90:0/1045613556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4101920 msgr2=0x7f77e4105960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/1045613556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4101920 0x7f77e4105960 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f77d8009990 tx=0x7f77d8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 -- 172.21.15.90:0/1045613556 shutdown_connections 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/1045613556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e4106090 0x7f77e410dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/1045613556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4100f70 0x7f77e4101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/1045613556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4101920 0x7f77e4105960 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 -- 172.21.15.90:0/1045613556 >> 172.21.15.90:0/1045613556 conn(0x7f77e40fc7f0 msgr2=0x7f77e40fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 -- 172.21.15.90:0/1045613556 shutdown_connections 2024-04-06T14:05:27.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.777+0000 7f77eb6f0700 1 -- 172.21.15.90:0/1045613556 wait complete. 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 Processor -- start 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 -- start start 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4106090 0x7f77e4197720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e419bac0 0x7f77e419bf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419c540 con 0x7f77e4100f70 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419c6b0 con 0x7f77e4106090 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77eb6f0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77e419c870 con 0x7f77e419bac0 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50250/0 (socket says 172.21.15.90:50250) 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 -- 172.21.15.90:0/776667042 learned_addr learned my addr 172.21.15.90:0/776667042 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e8c8b700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4106090 0x7f77e4197720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e9c8d700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e419bac0 0x7f77e419bf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 -- 172.21.15.90:0/776667042 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e419bac0 msgr2=0x7f77e419bf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e419bac0 0x7f77e419bf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 -- 172.21.15.90:0/776667042 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4106090 msgr2=0x7f77e4197720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4106090 0x7f77e4197720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.778+0000 7f77e948c700 1 -- 172.21.15.90:0/776667042 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77d80095f0 con 0x7f77e4100f70 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77e948c700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f77e000e9a0 tx=0x7f77e000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77e0009790 con 0x7f77e4100f70 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f77e41a3dd0 con 0x7f77e4100f70 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f77e0010430 con 0x7f77e4100f70 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f77e0018680 con 0x7f77e4100f70 2024-04-06T14:05:27.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.779+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f77e41a4200 con 0x7f77e4100f70 2024-04-06T14:05:27.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.780+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f77e4109360 con 0x7f77e4100f70 2024-04-06T14:05:27.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.782+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 22) v1 ==== 136044+0+0 (secure 0 0 0) 0x7f77e00105a0 con 0x7f77e4100f70 2024-04-06T14:05:27.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.782+0000 7f77d67fc700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 0x7f77d00a37e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:27.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.782+0000 7f77e8c8b700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 0x7f77d00a37e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:27.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.782+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f77e0014070 con 0x7f77e4100f70 2024-04-06T14:05:27.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.783+0000 7f77e8c8b700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 0x7f77d00a37e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f77d800b2e0 tx=0x7f77d8005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:27.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.786+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f77e00964e0 con 0x7f77e4100f70 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi105:/etc/ceph/ceph.conf 2024-04-06T14:05:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: mgrmap e22: x(active, since 3s), standbys: z, y 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1407063443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:27.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:27 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/435637717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:27.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.981+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f77e41984e0 con 0x7f77e4100f70 2024-04-06T14:05:27.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.982+0000 7f77d67fc700 1 -- 172.21.15.90:0/776667042 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v84) v1 ==== 100+0+11 (secure 0 0 0) 0x7f77e0099b00 con 0x7f77e4100f70 2024-04-06T14:05:27.983 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:05:27.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.983+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 msgr2=0x7f77d00a37e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 0x7f77d00a37e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f77d800b2e0 tx=0x7f77d8005960 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 msgr2=0x7f77e41971e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f77e000e9a0 tx=0x7f77e000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 shutdown_connections 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f77d00a1320 0x7f77d00a37e0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f77e419bac0 0x7f77e419bf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f77e4100f70 0x7f77e41971e0 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 --2- 172.21.15.90:0/776667042 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f77e4106090 0x7f77e4197720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 >> 172.21.15.90:0/776667042 conn(0x7f77e40fc7f0 msgr2=0x7f77e40fdb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 shutdown_connections 2024-04-06T14:05:27.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:27.984+0000 7f77eb6f0700 1 -- 172.21.15.90:0/776667042 wait complete. 2024-04-06T14:05:27.995 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:05:27.995 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:05:27.995 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi105:/etc/ceph/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.996 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: mgrmap e22: x(active, since 3s), standbys: z, y 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1407063443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:27.997 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:27 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/435637717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:27.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:05:27.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:05:27.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi090:/etc/ceph/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi105:/etc/ceph/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi134:/etc/ceph/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi134:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: Updating smithi090:/etc/ceph/ceph.client.admin.keyring 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: mgrmap e22: x(active, since 3s), standbys: z, y 2024-04-06T14:05:27.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1407063443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:05:28.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:27 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/435637717' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.001 INFO:tasks.fwd_scrub:Wait for all MDSs to reach steady state... 2024-04-06T14:05:28.001 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:05:28.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 -- 172.21.15.90:0/2316736184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c102750 msgr2=0x7f678c10ac30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 --2- 172.21.15.90:0/2316736184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c102750 0x7f678c10ac30 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f6780009990 tx=0x7f6780009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 -- 172.21.15.90:0/2316736184 shutdown_connections 2024-04-06T14:05:28.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 --2- 172.21.15.90:0/2316736184 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c10d680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 --2- 172.21.15.90:0/2316736184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c101e30 0x7f678c102210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 --2- 172.21.15.90:0/2316736184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c102750 0x7f678c10ac30 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 -- 172.21.15.90:0/2316736184 >> 172.21.15.90:0/2316736184 conn(0x7f678c0fb5f0 msgr2=0x7f678c0fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 -- 172.21.15.90:0/2316736184 shutdown_connections 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.122+0000 7f67918d4700 1 -- 172.21.15.90:0/2316736184 wait complete. 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 Processor -- start 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 -- start start 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c102750 0x7f678c197570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c19b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f678c19bf50 con 0x7f678c101e30 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f678c19c0c0 con 0x7f678c102750 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f67918d4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f678c19c280 con 0x7f678c10b200 2024-04-06T14:05:28.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f678affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f678affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50264/0 (socket says 172.21.15.90:50264) 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.123+0000 7f678affd700 1 -- 172.21.15.90:0/1226175357 learned_addr learned my addr 172.21.15.90:0/1226175357 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678a7fc700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c102750 0x7f678c197570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678b7fe700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c19b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 -- 172.21.15.90:0/1226175357 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 msgr2=0x7f678c19b910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c19b910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 -- 172.21.15.90:0/1226175357 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c102750 msgr2=0x7f678c197570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c102750 0x7f678c197570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 -- 172.21.15.90:0/1226175357 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f67800095f0 con 0x7f678c101e30 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678b7fe700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c19b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f678affd700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f677c00e9a0 tx=0x7f677c00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f677c009790 con 0x7f678c101e30 2024-04-06T14:05:28.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f678c19c560 con 0x7f678c101e30 2024-04-06T14:05:28.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f677c00cbd0 con 0x7f678c101e30 2024-04-06T14:05:28.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f677c018680 con 0x7f678c101e30 2024-04-06T14:05:28.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.124+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f678c1a3da0 con 0x7f678c101e30 2024-04-06T14:05:28.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.125+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f678c106320 con 0x7f678c101e30 2024-04-06T14:05:28.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.126+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 22) v1 ==== 136044+0+0 (secure 0 0 0) 0x7f677c00cd40 con 0x7f678c101e30 2024-04-06T14:05:28.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.127+0000 7f67908d2700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 0x7f67740a37e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.127+0000 7f678a7fc700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 0x7f67740a37e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.127+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f677c014070 con 0x7f678c101e30 2024-04-06T14:05:28.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.127+0000 7f678a7fc700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 0x7f67740a37e0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f67800098e0 tx=0x7f6780005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.130+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f677c0966e0 con 0x7f678c101e30 2024-04-06T14:05:28.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.373+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f678c068660 con 0x7f678c101e30 2024-04-06T14:05:28.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.374+0000 7f67908d2700 1 -- 172.21.15.90:0/1226175357 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f677c099d00 con 0x7f678c101e30 2024-04-06T14:05:28.374 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:28.375 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 msgr2=0x7f67740a37e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 0x7f67740a37e0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f67800098e0 tx=0x7f6780005960 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 msgr2=0x7f678c197030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7f677c00e9a0 tx=0x7f677c00ecb0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 shutdown_connections 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f67740a1320 0x7f67740a37e0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f678c10b200 0x7f678c19b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f678c101e30 0x7f678c197030 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 --2- 172.21.15.90:0/1226175357 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f678c102750 0x7f678c197570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 >> 172.21.15.90:0/1226175357 conn(0x7f678c0fb5f0 msgr2=0x7f678c0fcf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:28.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 shutdown_connections 2024-04-06T14:05:28.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.376+0000 7f67918d4700 1 -- 172.21.15.90:0/1226175357 wait complete. 2024-04-06T14:05:28.378 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:05:28.389 INFO:tasks.fwd_scrub:Ready to start scrub thrashing 2024-04-06T14:05:28.389 INFO:tasks.fwd_scrub.ceph_manager:waiting for clean 2024-04-06T14:05:28.389 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-06T14:05:28.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 -- 172.21.15.90:0/3156118927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830100480 msgr2=0x7fd83010c990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 --2- 172.21.15.90:0/3156118927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830100480 0x7fd83010c990 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fd82800b0a0 tx=0x7fd82800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 -- 172.21.15.90:0/3156118927 shutdown_connections 2024-04-06T14:05:28.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 --2- 172.21.15.90:0/3156118927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830100480 0x7fd83010c990 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 --2- 172.21.15.90:0/3156118927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd830105ad0 0x7fd830105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 --2- 172.21.15.90:0/3156118927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd8300ffac0 0x7fd8300fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 -- 172.21.15.90:0/3156118927 >> 172.21.15.90:0/3156118927 conn(0x7fd8300fb590 msgr2=0x7fd8300fd9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 -- 172.21.15.90:0/3156118927 shutdown_connections 2024-04-06T14:05:28.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.521+0000 7fd82f59e700 1 -- 172.21.15.90:0/3156118927 wait complete. 2024-04-06T14:05:28.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 Processor -- start 2024-04-06T14:05:28.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 -- start start 2024-04-06T14:05:28.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd830100480 0x7fd830197610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830105ad0 0x7fd83019b9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd83019bff0 con 0x7fd8300ffac0 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd83019c160 con 0x7fd830105ad0 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82f59e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd83019c320 con 0x7fd830100480 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.522+0000 7fd82e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50290/0 (socket says 172.21.15.90:50290) 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 -- 172.21.15.90:0/2979843421 learned_addr learned my addr 172.21.15.90:0/2979843421 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82ed9d700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830105ad0 0x7fd83019b9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82dd9b700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd830100480 0x7fd830197610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 -- 172.21.15.90:0/2979843421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd830100480 msgr2=0x7fd830197610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd830100480 0x7fd830197610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 -- 172.21.15.90:0/2979843421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830105ad0 msgr2=0x7fd83019b9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830105ad0 0x7fd83019b9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 -- 172.21.15.90:0/2979843421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd828009d00 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.523+0000 7fd82e59c700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7fd82000b530 tx=0x7fd82000b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.524+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd820011830 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.524+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd820011e50 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.524+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd83019c600 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.524+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd82000f2e0 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.524+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd8301a3e40 con 0x7fd8300ffac0 2024-04-06T14:05:28.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.525+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd8301090f0 con 0x7fd8300ffac0 2024-04-06T14:05:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.526+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fd8200105a0 con 0x7fd8300ffac0 2024-04-06T14:05:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.526+0000 7fd81f7fe700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 0x7fd8180a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.526+0000 7fd82dd9b700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 0x7fd8180a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.527+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fd8200cc890 con 0x7fd8300ffac0 2024-04-06T14:05:28.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.527+0000 7fd82dd9b700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 0x7fd8180a3870 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fd830198620 tx=0x7fd824009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.531+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd820095bb0 con 0x7fd8300ffac0 2024-04-06T14:05:28.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.728+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fd8301983d0 con 0x7fd8180a13b0 2024-04-06T14:05:28.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.739+0000 7fd81f7fe700 1 -- 172.21.15.90:0/2979843421 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+344496 (secure 0 0 0) 0x7fd8301983d0 con 0x7fd8180a13b0 2024-04-06T14:05:28.744 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:28.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 msgr2=0x7fd8180a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 0x7fd8180a3870 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fd830198620 tx=0x7fd824009190 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 msgr2=0x7fd8301970d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 crypto rx=0x7fd82000b530 tx=0x7fd82000b840 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 shutdown_connections 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd8180a13b0 0x7fd8180a3870 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd830105ad0 0x7fd83019b9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd8300ffac0 0x7fd8301970d0 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 --2- 172.21.15.90:0/2979843421 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd830100480 0x7fd830197610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 >> 172.21.15.90:0/2979843421 conn(0x7fd8300fb590 msgr2=0x7fd830069430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 shutdown_connections 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.742+0000 7fd82f59e700 1 -- 172.21.15.90:0/2979843421 wait complete. 2024-04-06T14:05:28.747 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3320543710' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/776667042' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1226175357' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:28.754 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:28 smithi090 ceph-mon[43420]: mgrmap e23: x(active, since 4s), standbys: z, y 2024-04-06T14:05:28.755 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":5,"stamp":"2024-04-06T14:05:27.571371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1187895,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":727,"num_read_kb":518,"num_write":452,"num_write_kb":4368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1181318,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":854,"ondisk_log_size":854,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":360068,"kb_used_data":11268,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124581756,"statfs":{"total":1151940427776,"available":1151571718144,"internally_reserved":0,"allocated":11538432,"data_stored":8884044,"data_compressed":223581,"data_compressed_allocated":294912,"data_compressed_original":3440640,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.988099"},"pg_stats":[{"pgid":"3.3f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615419+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:05:23.615419+0000","last_peered":"2024-04-06T14:05:23.615419+0000","last_clean":"2024-04-06T14:05:23.615419+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:05:23.615419+0000","last_undegraded":"2024-04-06T14:05:23.615419+0000","last_fullsized":"2024-04-06T14:05:23.615419+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607957+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:05:23.607957+0000","last_peered":"2024-04-06T14:05:23.607957+0000","last_clean":"2024-04-06T14:05:23.607957+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:05:23.607957+0000","last_undegraded":"2024-04-06T14:05:23.607957+0000","last_fullsized":"2024-04-06T14:05:23.607957+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616328+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:05:23.616328+0000","last_peered":"2024-04-06T14:05:23.616328+0000","last_clean":"2024-04-06T14:05:23.616328+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:05:23.616328+0000","last_undegraded":"2024-04-06T14:05:23.616328+0000","last_fullsized":"2024-04-06T14:05:23.616328+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.593301+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:05:23.593301+0000","last_peered":"2024-04-06T14:05:23.593301+0000","last_clean":"2024-04-06T14:05:23.593301+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:05:23.593301+0000","last_undegraded":"2024-04-06T14:05:23.593301+0000","last_fullsized":"2024-04-06T14:05:23.593301+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.592862+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:05:23.592862+0000","last_peered":"2024-04-06T14:05:23.592862+0000","last_clean":"2024-04-06T14:05:23.592862+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:05:23.592862+0000","last_undegraded":"2024-04-06T14:05:23.592862+0000","last_fullsized":"2024-04-06T14:05:23.592862+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604597+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:05:23.604597+0000","last_peered":"2024-04-06T14:05:23.604597+0000","last_clean":"2024-04-06T14:05:23.604597+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:05:23.604597+0000","last_undegraded":"2024-04-06T14:05:23.604597+0000","last_fullsized":"2024-04-06T14:05:23.604597+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609927+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:05:23.609927+0000","last_peered":"2024-04-06T14:05:23.609927+0000","last_clean":"2024-04-06T14:05:23.609927+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:05:23.609927+0000","last_undegraded":"2024-04-06T14:05:23.609927+0000","last_fullsized":"2024-04-06T14:05:23.609927+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603861+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:05:23.603861+0000","last_peered":"2024-04-06T14:05:23.603861+0000","last_clean":"2024-04-06T14:05:23.603861+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:05:23.603861+0000","last_undegraded":"2024-04-06T14:05:23.603861+0000","last_fullsized":"2024-04-06T14:05:23.603861+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603421+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:05:23.603421+0000","last_peered":"2024-04-06T14:05:23.603421+0000","last_clean":"2024-04-06T14:05:23.603421+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:05:23.603421+0000","last_undegraded":"2024-04-06T14:05:23.603421+0000","last_fullsized":"2024-04-06T14:05:23.603421+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"84'16","reported_seq":30,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603913+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:05:23.603913+0000","last_peered":"2024-04-06T14:05:23.603913+0000","last_clean":"2024-04-06T14:05:23.603913+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:05:23.603913+0000","last_undegraded":"2024-04-06T14:05:23.603913+0000","last_fullsized":"2024-04-06T14:05:23.603913+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609304+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:05:23.609304+0000","last_peered":"2024-04-06T14:05:23.609304+0000","last_clean":"2024-04-06T14:05:23.609304+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:05:23.609304+0000","last_undegraded":"2024-04-06T14:05:23.609304+0000","last_fullsized":"2024-04-06T14:05:23.609304+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602535+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:05:23.602535+0000","last_peered":"2024-04-06T14:05:23.602535+0000","last_clean":"2024-04-06T14:05:23.602535+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:05:23.602535+0000","last_undegraded":"2024-04-06T14:05:23.602535+0000","last_fullsized":"2024-04-06T14:05:23.602535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610171+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:05:23.610171+0000","last_peered":"2024-04-06T14:05:23.610171+0000","last_clean":"2024-04-06T14:05:23.610171+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:05:23.610171+0000","last_undegraded":"2024-04-06T14:05:23.610171+0000","last_fullsized":"2024-04-06T14:05:23.610171+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611310+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:05:23.611310+0000","last_peered":"2024-04-06T14:05:23.611310+0000","last_clean":"2024-04-06T14:05:23.611310+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:05:23.611310+0000","last_undegraded":"2024-04-06T14:05:23.611310+0000","last_fullsized":"2024-04-06T14:05:23.611310+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614994+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:05:23.614994+0000","last_peered":"2024-04-06T14:05:23.614994+0000","last_clean":"2024-04-06T14:05:23.614994+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:05:23.614994+0000","last_undegraded":"2024-04-06T14:05:23.614994+0000","last_fullsized":"2024-04-06T14:05:23.614994+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611373+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:05:23.611373+0000","last_peered":"2024-04-06T14:05:23.611373+0000","last_clean":"2024-04-06T14:05:23.611373+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:05:23.611373+0000","last_undegraded":"2024-04-06T14:05:23.611373+0000","last_fullsized":"2024-04-06T14:05:23.611373+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615524+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:05:23.615524+0000","last_peered":"2024-04-06T14:05:23.615524+0000","last_clean":"2024-04-06T14:05:23.615524+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:05:23.615524+0000","last_undegraded":"2024-04-06T14:05:23.615524+0000","last_fullsized":"2024-04-06T14:05:23.615524+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613532+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:05:23.613532+0000","last_peered":"2024-04-06T14:05:23.613532+0000","last_clean":"2024-04-06T14:05:23.613532+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:05:23.613532+0000","last_undegraded":"2024-04-06T14:05:23.613532+0000","last_fullsized":"2024-04-06T14:05:23.613532+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611801+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:05:23.611801+0000","last_peered":"2024-04-06T14:05:23.611801+0000","last_clean":"2024-04-06T14:05:23.611801+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:05:23.611801+0000","last_undegraded":"2024-04-06T14:05:23.611801+0000","last_fullsized":"2024-04-06T14:05:23.611801+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":44,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.235666+0000","last_change":"2024-04-06T14:05:05.235370+0000","last_active":"2024-04-06T14:05:05.235666+0000","last_peered":"2024-04-06T14:05:05.235666+0000","last_clean":"2024-04-06T14:05:05.235666+0000","last_became_active":"2024-04-06T14:05:05.229253+0000","last_became_peered":"2024-04-06T14:05:05.229253+0000","last_unstale":"2024-04-06T14:05:05.235666+0000","last_undegraded":"2024-04-06T14:05:05.235666+0000","last_fullsized":"2024-04-06T14:05:05.235666+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:37:23.805018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,4],"acting":[9,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596468+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:05:23.596468+0000","last_peered":"2024-04-06T14:05:23.596468+0000","last_clean":"2024-04-06T14:05:23.596468+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:05:23.596468+0000","last_undegraded":"2024-04-06T14:05:23.596468+0000","last_fullsized":"2024-04-06T14:05:23.596468+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190339+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:05:05.190339+0000","last_peered":"2024-04-06T14:05:05.190339+0000","last_clean":"2024-04-06T14:05:05.190339+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:05:05.190339+0000","last_undegraded":"2024-04-06T14:05:05.190339+0000","last_fullsized":"2024-04-06T14:05:05.190339+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617749+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:05:23.617749+0000","last_peered":"2024-04-06T14:05:23.617749+0000","last_clean":"2024-04-06T14:05:23.617749+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:05:23.617749+0000","last_undegraded":"2024-04-06T14:05:23.617749+0000","last_fullsized":"2024-04-06T14:05:23.617749+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196231+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:05:05.196231+0000","last_peered":"2024-04-06T14:05:05.196231+0000","last_clean":"2024-04-06T14:05:05.196231+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:05:05.196231+0000","last_undegraded":"2024-04-06T14:05:05.196231+0000","last_fullsized":"2024-04-06T14:05:05.196231+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607848+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:05:23.607848+0000","last_peered":"2024-04-06T14:05:23.607848+0000","last_clean":"2024-04-06T14:05:23.607848+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:05:23.607848+0000","last_undegraded":"2024-04-06T14:05:23.607848+0000","last_fullsized":"2024-04-06T14:05:23.607848+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195822+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:05:05.195822+0000","last_peered":"2024-04-06T14:05:05.195822+0000","last_clean":"2024-04-06T14:05:05.195822+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:05:05.195822+0000","last_undegraded":"2024-04-06T14:05:05.195822+0000","last_fullsized":"2024-04-06T14:05:05.195822+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"84'15","reported_seq":38,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600996+0000","last_change":"2024-04-06T14:05:05.196872+0000","last_active":"2024-04-06T14:05:23.600996+0000","last_peered":"2024-04-06T14:05:23.600996+0000","last_clean":"2024-04-06T14:05:23.600996+0000","last_became_active":"2024-04-06T14:05:05.196522+0000","last_became_peered":"2024-04-06T14:05:05.196522+0000","last_unstale":"2024-04-06T14:05:23.600996+0000","last_undegraded":"2024-04-06T14:05:23.600996+0000","last_fullsized":"2024-04-06T14:05:23.600996+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:12:54.386682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,1],"acting":[5,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"84'6","reported_seq":101,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626416+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:05:23.626416+0000","last_peered":"2024-04-06T14:05:23.626416+0000","last_clean":"2024-04-06T14:05:23.626416+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:05:23.626416+0000","last_undegraded":"2024-04-06T14:05:23.626416+0000","last_fullsized":"2024-04-06T14:05:23.626416+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":967,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607842+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:05:23.607842+0000","last_peered":"2024-04-06T14:05:23.607842+0000","last_clean":"2024-04-06T14:05:23.607842+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:05:23.607842+0000","last_undegraded":"2024-04-06T14:05:23.607842+0000","last_fullsized":"2024-04-06T14:05:23.607842+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.259157+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:05:05.259157+0000","last_peered":"2024-04-06T14:05:05.259157+0000","last_clean":"2024-04-06T14:05:05.259157+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:05:05.259157+0000","last_undegraded":"2024-04-06T14:05:05.259157+0000","last_fullsized":"2024-04-06T14:05:05.259157+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.599735+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:05:23.599735+0000","last_peered":"2024-04-06T14:05:23.599735+0000","last_clean":"2024-04-06T14:05:23.599735+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:05:23.599735+0000","last_undegraded":"2024-04-06T14:05:23.599735+0000","last_fullsized":"2024-04-06T14:05:23.599735+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193183+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:05:05.193183+0000","last_peered":"2024-04-06T14:05:05.193183+0000","last_clean":"2024-04-06T14:05:05.193183+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:05:05.193183+0000","last_undegraded":"2024-04-06T14:05:05.193183+0000","last_fullsized":"2024-04-06T14:05:05.193183+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604741+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:05:23.604741+0000","last_peered":"2024-04-06T14:05:23.604741+0000","last_clean":"2024-04-06T14:05:23.604741+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:05:23.604741+0000","last_undegraded":"2024-04-06T14:05:23.604741+0000","last_fullsized":"2024-04-06T14:05:23.604741+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189600+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:05:05.189600+0000","last_peered":"2024-04-06T14:05:05.189600+0000","last_clean":"2024-04-06T14:05:05.189600+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:05:05.189600+0000","last_undegraded":"2024-04-06T14:05:05.189600+0000","last_fullsized":"2024-04-06T14:05:05.189600+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607641+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:05:23.607641+0000","last_peered":"2024-04-06T14:05:23.607641+0000","last_clean":"2024-04-06T14:05:23.607641+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:05:23.607641+0000","last_undegraded":"2024-04-06T14:05:23.607641+0000","last_fullsized":"2024-04-06T14:05:23.607641+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"84'4","reported_seq":97,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626088+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:05:23.626088+0000","last_peered":"2024-04-06T14:05:23.626088+0000","last_clean":"2024-04-06T14:05:23.626088+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:05:23.626088+0000","last_undegraded":"2024-04-06T14:05:23.626088+0000","last_fullsized":"2024-04-06T14:05:23.626088+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614466+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:05:23.614466+0000","last_peered":"2024-04-06T14:05:23.614466+0000","last_clean":"2024-04-06T14:05:23.614466+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:05:23.614466+0000","last_undegraded":"2024-04-06T14:05:23.614466+0000","last_fullsized":"2024-04-06T14:05:23.614466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194821+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:05:05.194821+0000","last_peered":"2024-04-06T14:05:05.194821+0000","last_clean":"2024-04-06T14:05:05.194821+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:05:05.194821+0000","last_undegraded":"2024-04-06T14:05:05.194821+0000","last_fullsized":"2024-04-06T14:05:05.194821+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610283+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:05:23.610283+0000","last_peered":"2024-04-06T14:05:23.610283+0000","last_clean":"2024-04-06T14:05:23.610283+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:05:23.610283+0000","last_undegraded":"2024-04-06T14:05:23.610283+0000","last_fullsized":"2024-04-06T14:05:23.610283+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190092+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:05:05.190092+0000","last_peered":"2024-04-06T14:05:05.190092+0000","last_clean":"2024-04-06T14:05:05.190092+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:05:05.190092+0000","last_undegraded":"2024-04-06T14:05:05.190092+0000","last_fullsized":"2024-04-06T14:05:05.190092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604056+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:05:23.604056+0000","last_peered":"2024-04-06T14:05:23.604056+0000","last_clean":"2024-04-06T14:05:23.604056+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:05:23.604056+0000","last_undegraded":"2024-04-06T14:05:23.604056+0000","last_fullsized":"2024-04-06T14:05:23.604056+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"84'34","reported_seq":215,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.840919+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:05:23.840919+0000","last_peered":"2024-04-06T14:05:23.840919+0000","last_clean":"2024-04-06T14:05:23.840919+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:05:23.840919+0000","last_undegraded":"2024-04-06T14:05:23.840919+0000","last_fullsized":"2024-04-06T14:05:23.840919+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67722,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":166,"num_read_kb":143,"num_write":35,"num_write_kb":80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605924+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:05:23.605924+0000","last_peered":"2024-04-06T14:05:23.605924+0000","last_clean":"2024-04-06T14:05:23.605924+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:05:23.605924+0000","last_undegraded":"2024-04-06T14:05:23.605924+0000","last_fullsized":"2024-04-06T14:05:23.605924+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":22,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191794+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:05:05.191794+0000","last_peered":"2024-04-06T14:05:05.191794+0000","last_clean":"2024-04-06T14:05:05.191794+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:05:05.191794+0000","last_undegraded":"2024-04-06T14:05:05.191794+0000","last_fullsized":"2024-04-06T14:05:05.191794+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612189+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:05:23.612189+0000","last_peered":"2024-04-06T14:05:23.612189+0000","last_clean":"2024-04-06T14:05:23.612189+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:05:23.612189+0000","last_undegraded":"2024-04-06T14:05:23.612189+0000","last_fullsized":"2024-04-06T14:05:23.612189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189874+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:05:05.189874+0000","last_peered":"2024-04-06T14:05:05.189874+0000","last_clean":"2024-04-06T14:05:05.189874+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:05:05.189874+0000","last_undegraded":"2024-04-06T14:05:05.189874+0000","last_fullsized":"2024-04-06T14:05:05.189874+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610716+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:05:23.610716+0000","last_peered":"2024-04-06T14:05:23.610716+0000","last_clean":"2024-04-06T14:05:23.610716+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:05:23.610716+0000","last_undegraded":"2024-04-06T14:05:23.610716+0000","last_fullsized":"2024-04-06T14:05:23.610716+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"84'6","reported_seq":104,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.629802+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:05:23.629802+0000","last_peered":"2024-04-06T14:05:23.629802+0000","last_clean":"2024-04-06T14:05:23.629802+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:05:23.629802+0000","last_undegraded":"2024-04-06T14:05:23.629802+0000","last_fullsized":"2024-04-06T14:05:23.629802+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1388,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":84,"num_read_kb":6,"num_write":5,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605694+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:05:23.605694+0000","last_peered":"2024-04-06T14:05:23.605694+0000","last_clean":"2024-04-06T14:05:23.605694+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:05:23.605694+0000","last_undegraded":"2024-04-06T14:05:23.605694+0000","last_fullsized":"2024-04-06T14:05:23.605694+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"81'15","reported_seq":31,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195316+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:05:05.195316+0000","last_peered":"2024-04-06T14:05:05.195316+0000","last_clean":"2024-04-06T14:05:05.195316+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:05:05.195316+0000","last_undegraded":"2024-04-06T14:05:05.195316+0000","last_fullsized":"2024-04-06T14:05:05.195316+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602036+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:05:23.602036+0000","last_peered":"2024-04-06T14:05:23.602036+0000","last_clean":"2024-04-06T14:05:23.602036+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:05:23.602036+0000","last_undegraded":"2024-04-06T14:05:23.602036+0000","last_fullsized":"2024-04-06T14:05:23.602036+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'2","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196332+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:05:05.196332+0000","last_peered":"2024-04-06T14:05:05.196332+0000","last_clean":"2024-04-06T14:05:05.196332+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:05:05.196332+0000","last_undegraded":"2024-04-06T14:05:05.196332+0000","last_fullsized":"2024-04-06T14:05:05.196332+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604329+0000","last_change":"2024-04-06T14:05:05.196327+0000","last_active":"2024-04-06T14:05:23.604329+0000","last_peered":"2024-04-06T14:05:23.604329+0000","last_clean":"2024-04-06T14:05:23.604329+0000","last_became_active":"2024-04-06T14:05:05.195845+0000","last_became_peered":"2024-04-06T14:05:05.195845+0000","last_unstale":"2024-04-06T14:05:23.604329+0000","last_undegraded":"2024-04-06T14:05:23.604329+0000","last_fullsized":"2024-04-06T14:05:23.604329+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:16:06.262603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,6],"acting":[11,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191665+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:05:05.191665+0000","last_peered":"2024-04-06T14:05:05.191665+0000","last_clean":"2024-04-06T14:05:05.191665+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:05:05.191665+0000","last_undegraded":"2024-04-06T14:05:05.191665+0000","last_fullsized":"2024-04-06T14:05:05.191665+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.597349+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:05:23.597349+0000","last_peered":"2024-04-06T14:05:23.597349+0000","last_clean":"2024-04-06T14:05:23.597349+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:05:23.597349+0000","last_undegraded":"2024-04-06T14:05:23.597349+0000","last_fullsized":"2024-04-06T14:05:23.597349+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194517+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:05:05.194517+0000","last_peered":"2024-04-06T14:05:05.194517+0000","last_clean":"2024-04-06T14:05:05.194517+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:05:05.194517+0000","last_undegraded":"2024-04-06T14:05:05.194517+0000","last_fullsized":"2024-04-06T14:05:05.194517+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613134+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:05:23.613134+0000","last_peered":"2024-04-06T14:05:23.613134+0000","last_clean":"2024-04-06T14:05:23.613134+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:05:23.613134+0000","last_undegraded":"2024-04-06T14:05:23.613134+0000","last_fullsized":"2024-04-06T14:05:23.613134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604058+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:05:23.604058+0000","last_peered":"2024-04-06T14:05:23.604058+0000","last_clean":"2024-04-06T14:05:23.604058+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:05:23.604058+0000","last_undegraded":"2024-04-06T14:05:23.604058+0000","last_fullsized":"2024-04-06T14:05:23.604058+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"83'5","reported_seq":98,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.576804+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:05:23.576804+0000","last_peered":"2024-04-06T14:05:23.576804+0000","last_clean":"2024-04-06T14:05:23.576804+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:05:23.576804+0000","last_undegraded":"2024-04-06T14:05:23.576804+0000","last_fullsized":"2024-04-06T14:05:23.576804+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604005+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:05:23.604005+0000","last_peered":"2024-04-06T14:05:23.604005+0000","last_clean":"2024-04-06T14:05:23.604005+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:05:23.604005+0000","last_undegraded":"2024-04-06T14:05:23.604005+0000","last_fullsized":"2024-04-06T14:05:23.604005+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228303+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:05:05.228303+0000","last_peered":"2024-04-06T14:05:05.228303+0000","last_clean":"2024-04-06T14:05:05.228303+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:05:05.228303+0000","last_undegraded":"2024-04-06T14:05:05.228303+0000","last_fullsized":"2024-04-06T14:05:05.228303+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":337,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194265+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:05:05.194265+0000","last_peered":"2024-04-06T14:05:05.194265+0000","last_clean":"2024-04-06T14:05:05.194265+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:05:05.194265+0000","last_undegraded":"2024-04-06T14:05:05.194265+0000","last_fullsized":"2024-04-06T14:05:05.194265+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610576+0000","last_change":"2024-04-06T14:05:05.195151+0000","last_active":"2024-04-06T14:05:23.610576+0000","last_peered":"2024-04-06T14:05:23.610576+0000","last_clean":"2024-04-06T14:05:23.610576+0000","last_became_active":"2024-04-06T14:05:05.194828+0000","last_became_peered":"2024-04-06T14:05:05.194828+0000","last_unstale":"2024-04-06T14:05:23.610576+0000","last_undegraded":"2024-04-06T14:05:23.610576+0000","last_fullsized":"2024-04-06T14:05:23.610576+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:11:32.067936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,2],"acting":[6,9,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194363+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:05:05.194363+0000","last_peered":"2024-04-06T14:05:05.194363+0000","last_clean":"2024-04-06T14:05:05.194363+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:05:05.194363+0000","last_undegraded":"2024-04-06T14:05:05.194363+0000","last_fullsized":"2024-04-06T14:05:05.194363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"81'1","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196094+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:05:05.196094+0000","last_peered":"2024-04-06T14:05:05.196094+0000","last_clean":"2024-04-06T14:05:05.196094+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:05:05.196094+0000","last_undegraded":"2024-04-06T14:05:05.196094+0000","last_fullsized":"2024-04-06T14:05:05.196094+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609695+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:05:23.609695+0000","last_peered":"2024-04-06T14:05:23.609695+0000","last_clean":"2024-04-06T14:05:23.609695+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:05:23.609695+0000","last_undegraded":"2024-04-06T14:05:23.609695+0000","last_fullsized":"2024-04-06T14:05:23.609695+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"84'39","reported_seq":77,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.797513+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:05:23.797513+0000","last_peered":"2024-04-06T14:05:23.797513+0000","last_clean":"2024-04-06T14:05:23.797513+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:05:23.797513+0000","last_undegraded":"2024-04-06T14:05:23.797513+0000","last_fullsized":"2024-04-06T14:05:23.797513+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":68,"num_write_kb":833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"84'5","reported_seq":17,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616610+0000","last_change":"2024-04-06T14:05:05.196110+0000","last_active":"2024-04-06T14:05:23.616610+0000","last_peered":"2024-04-06T14:05:23.616610+0000","last_clean":"2024-04-06T14:05:23.616610+0000","last_became_active":"2024-04-06T14:05:05.195775+0000","last_became_peered":"2024-04-06T14:05:05.195775+0000","last_unstale":"2024-04-06T14:05:23.616610+0000","last_undegraded":"2024-04-06T14:05:23.616610+0000","last_fullsized":"2024-04-06T14:05:23.616610+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:12:33.956229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,7],"acting":[9,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228555+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:05:05.228555+0000","last_peered":"2024-04-06T14:05:05.228555+0000","last_clean":"2024-04-06T14:05:05.228555+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:05:05.228555+0000","last_undegraded":"2024-04-06T14:05:05.228555+0000","last_fullsized":"2024-04-06T14:05:05.228555+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617648+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:05:23.617648+0000","last_peered":"2024-04-06T14:05:23.617648+0000","last_clean":"2024-04-06T14:05:23.617648+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:05:23.617648+0000","last_undegraded":"2024-04-06T14:05:23.617648+0000","last_fullsized":"2024-04-06T14:05:23.617648+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":30,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.200728+0000","last_change":"2024-04-06T14:05:05.200365+0000","last_active":"2024-04-06T14:05:05.200728+0000","last_peered":"2024-04-06T14:05:05.200728+0000","last_clean":"2024-04-06T14:05:05.200728+0000","last_became_active":"2024-04-06T14:05:05.196172+0000","last_became_peered":"2024-04-06T14:05:05.196172+0000","last_unstale":"2024-04-06T14:05:05.200728+0000","last_undegraded":"2024-04-06T14:05:05.200728+0000","last_fullsized":"2024-04-06T14:05:05.200728+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:44.928650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.575363+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:05:23.575363+0000","last_peered":"2024-04-06T14:05:23.575363+0000","last_clean":"2024-04-06T14:05:23.575363+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:05:23.575363+0000","last_undegraded":"2024-04-06T14:05:23.575363+0000","last_fullsized":"2024-04-06T14:05:23.575363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189911+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:05:05.189911+0000","last_peered":"2024-04-06T14:05:05.189911+0000","last_clean":"2024-04-06T14:05:05.189911+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:05:05.189911+0000","last_undegraded":"2024-04-06T14:05:05.189911+0000","last_fullsized":"2024-04-06T14:05:05.189911+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615526+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:05:23.615526+0000","last_peered":"2024-04-06T14:05:23.615526+0000","last_clean":"2024-04-06T14:05:23.615526+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:05:23.615526+0000","last_undegraded":"2024-04-06T14:05:23.615526+0000","last_fullsized":"2024-04-06T14:05:23.615526+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:06.497211+0000","last_change":"2024-04-06T14:05:05.200944+0000","last_active":"2024-04-06T14:05:06.497211+0000","last_peered":"2024-04-06T14:05:06.497211+0000","last_clean":"2024-04-06T14:05:06.497211+0000","last_became_active":"2024-04-06T14:05:05.196972+0000","last_became_peered":"2024-04-06T14:05:05.196972+0000","last_unstale":"2024-04-06T14:05:06.497211+0000","last_undegraded":"2024-04-06T14:05:06.497211+0000","last_fullsized":"2024-04-06T14:05:06.497211+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:00:15.481685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,3,8],"acting":[11,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600338+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:05:23.600338+0000","last_peered":"2024-04-06T14:05:23.600338+0000","last_clean":"2024-04-06T14:05:23.600338+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:05:23.600338+0000","last_undegraded":"2024-04-06T14:05:23.600338+0000","last_fullsized":"2024-04-06T14:05:23.600338+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195568+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:05:05.195568+0000","last_peered":"2024-04-06T14:05:05.195568+0000","last_clean":"2024-04-06T14:05:05.195568+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:05:05.195568+0000","last_undegraded":"2024-04-06T14:05:05.195568+0000","last_fullsized":"2024-04-06T14:05:05.195568+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612506+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:05:23.612506+0000","last_peered":"2024-04-06T14:05:23.612506+0000","last_clean":"2024-04-06T14:05:23.612506+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:05:23.612506+0000","last_undegraded":"2024-04-06T14:05:23.612506+0000","last_fullsized":"2024-04-06T14:05:23.612506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191823+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:05:05.191823+0000","last_peered":"2024-04-06T14:05:05.191823+0000","last_clean":"2024-04-06T14:05:05.191823+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:05:05.191823+0000","last_undegraded":"2024-04-06T14:05:05.191823+0000","last_fullsized":"2024-04-06T14:05:05.191823+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611707+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:05:23.611707+0000","last_peered":"2024-04-06T14:05:23.611707+0000","last_clean":"2024-04-06T14:05:23.611707+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:05:23.611707+0000","last_undegraded":"2024-04-06T14:05:23.611707+0000","last_fullsized":"2024-04-06T14:05:23.611707+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"84'39","reported_seq":62,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.794092+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:05:23.794092+0000","last_peered":"2024-04-06T14:05:23.794092+0000","last_clean":"2024-04-06T14:05:23.794092+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:05:23.794092+0000","last_undegraded":"2024-04-06T14:05:23.794092+0000","last_fullsized":"2024-04-06T14:05:23.794092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606466+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:05:23.606466+0000","last_peered":"2024-04-06T14:05:23.606466+0000","last_clean":"2024-04-06T14:05:23.606466+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:05:23.606466+0000","last_undegraded":"2024-04-06T14:05:23.606466+0000","last_fullsized":"2024-04-06T14:05:23.606466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193472+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:05:05.193472+0000","last_peered":"2024-04-06T14:05:05.193472+0000","last_clean":"2024-04-06T14:05:05.193472+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:05:05.193472+0000","last_undegraded":"2024-04-06T14:05:05.193472+0000","last_fullsized":"2024-04-06T14:05:05.193472+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.608430+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:05:23.608430+0000","last_peered":"2024-04-06T14:05:23.608430+0000","last_clean":"2024-04-06T14:05:23.608430+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:05:23.608430+0000","last_undegraded":"2024-04-06T14:05:23.608430+0000","last_fullsized":"2024-04-06T14:05:23.608430+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"84'9","reported_seq":32,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609331+0000","last_change":"2024-04-06T14:05:05.198436+0000","last_active":"2024-04-06T14:05:23.609331+0000","last_peered":"2024-04-06T14:05:23.609331+0000","last_clean":"2024-04-06T14:05:23.609331+0000","last_became_active":"2024-04-06T14:05:05.198028+0000","last_became_peered":"2024-04-06T14:05:05.198028+0000","last_unstale":"2024-04-06T14:05:23.609331+0000","last_undegraded":"2024-04-06T14:05:23.609331+0000","last_fullsized":"2024-04-06T14:05:23.609331+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:49:16.741498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"84'12","reported_seq":26,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604089+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:05:23.604089+0000","last_peered":"2024-04-06T14:05:23.604089+0000","last_clean":"2024-04-06T14:05:23.604089+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:05:23.604089+0000","last_undegraded":"2024-04-06T14:05:23.604089+0000","last_fullsized":"2024-04-06T14:05:23.604089+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615361+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:05:23.615361+0000","last_peered":"2024-04-06T14:05:23.615361+0000","last_clean":"2024-04-06T14:05:23.615361+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:05:23.615361+0000","last_undegraded":"2024-04-06T14:05:23.615361+0000","last_fullsized":"2024-04-06T14:05:23.615361+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615434+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:05:23.615434+0000","last_peered":"2024-04-06T14:05:23.615434+0000","last_clean":"2024-04-06T14:05:23.615434+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:05:23.615434+0000","last_undegraded":"2024-04-06T14:05:23.615434+0000","last_fullsized":"2024-04-06T14:05:23.615434+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615556+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:05:23.615556+0000","last_peered":"2024-04-06T14:05:23.615556+0000","last_clean":"2024-04-06T14:05:23.615556+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:05:23.615556+0000","last_undegraded":"2024-04-06T14:05:23.615556+0000","last_fullsized":"2024-04-06T14:05:23.615556+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604845+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:05:23.604845+0000","last_peered":"2024-04-06T14:05:23.604845+0000","last_clean":"2024-04-06T14:05:23.604845+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:05:23.604845+0000","last_undegraded":"2024-04-06T14:05:23.604845+0000","last_fullsized":"2024-04-06T14:05:23.604845+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600672+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:05:23.600672+0000","last_peered":"2024-04-06T14:05:23.600672+0000","last_clean":"2024-04-06T14:05:23.600672+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:05:23.600672+0000","last_undegraded":"2024-04-06T14:05:23.600672+0000","last_fullsized":"2024-04-06T14:05:23.600672+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596144+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:05:23.596144+0000","last_peered":"2024-04-06T14:05:23.596144+0000","last_clean":"2024-04-06T14:05:23.596144+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:05:23.596144+0000","last_undegraded":"2024-04-06T14:05:23.596144+0000","last_fullsized":"2024-04-06T14:05:23.596144+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606756+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:05:23.606756+0000","last_peered":"2024-04-06T14:05:23.606756+0000","last_clean":"2024-04-06T14:05:23.606756+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:05:23.606756+0000","last_undegraded":"2024-04-06T14:05:23.606756+0000","last_fullsized":"2024-04-06T14:05:23.606756+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606960+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:05:23.606960+0000","last_peered":"2024-04-06T14:05:23.606960+0000","last_clean":"2024-04-06T14:05:23.606960+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:05:23.606960+0000","last_undegraded":"2024-04-06T14:05:23.606960+0000","last_fullsized":"2024-04-06T14:05:23.606960+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613612+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:05:23.613612+0000","last_peered":"2024-04-06T14:05:23.613612+0000","last_clean":"2024-04-06T14:05:23.613612+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:05:23.613612+0000","last_undegraded":"2024-04-06T14:05:23.613612+0000","last_fullsized":"2024-04-06T14:05:23.613612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615353+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:05:23.615353+0000","last_peered":"2024-04-06T14:05:23.615353+0000","last_clean":"2024-04-06T14:05:23.615353+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:05:23.615353+0000","last_undegraded":"2024-04-06T14:05:23.615353+0000","last_fullsized":"2024-04-06T14:05:23.615353+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.598200+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:05:23.598200+0000","last_peered":"2024-04-06T14:05:23.598200+0000","last_clean":"2024-04-06T14:05:23.598200+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:05:23.598200+0000","last_undegraded":"2024-04-06T14:05:23.598200+0000","last_fullsized":"2024-04-06T14:05:23.598200+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":107160,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597293,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":591,"num_read_kb":399,"num_write":269,"num_write_kb":1428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":565248,"data_stored":1791879,"data_compressed":81405,"data_compressed_allocated":110592,"data_compressed_original":1572864,"omap_allocated":0,"internal_metadata":0},"log_size":215,"ondisk_log_size":215,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514523,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32000,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713152,"statfs":{"total":95995035648,"available":95962267648,"internally_reserved":0,"allocated":905216,"data_stored":444028,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.504},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.745,"15min":0.745},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.58199999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.436},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55700000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.55000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.57499999999999996}]}]},{"osd":10,"up_from":68,"seq":292057776157,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28072,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717080,"statfs":{"total":95995035648,"available":95966289920,"internally_reserved":0,"allocated":1077248,"data_stored":1235300,"data_compressed":46627,"data_compressed_allocated":61440,"data_compressed_original":729088,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.61099999999999999,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.311},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.51600000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.74399999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71399999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":0.65300000000000002},"min":{"1min":0.54600000000000004,"5min":0.501,"15min":0.501},"max":{"1min":0.84099999999999997,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67000000000000004,"15min":0.67300000000000004},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.84999999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.37}]},{"osd":6,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.68000000000000005,"15min":0.68799999999999994},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.75600000000000001,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.74099999999999999,"15min":0.71699999999999997},"min":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.46899999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.76400000000000001,"15min":0.76500000000000001},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.78500000000000003,"15min":0.79000000000000004},"min":{"1min":0.64900000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.92400000000000004,"5min":1.083,"15min":1.083},"last":0.51000000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.76000000000000001,"15min":0.76300000000000001},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":0.878,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76000000000000001,"15min":0.76600000000000001},"min":{"1min":0.58099999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.83199999999999996,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.53000000000000003}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.498},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.54600000000000004}]}]},{"osd":9,"up_from":63,"seq":270582939681,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":946176,"data_stored":837480,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,6,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40799999999999997,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.371},{"interface":"front","average":{"1min":0.371,"5min":0.44900000000000001,"15min":0.46200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.55600000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.314}]},{"osd":1,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.753,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75700000000000001,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.751,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.59599999999999997,"15min":0.60899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.74299999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.49399999999999999,"15min":0.50700000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.624,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69999999999999996,"5min":0.753,"15min":0.753},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88300000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.65600000000000003,"15min":0.67100000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.79400000000000004,"5min":1.107,"15min":1.107},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.63900000000000001,"15min":0.65200000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77800000000000002,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.57499999999999996,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72499999999999998,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.61699999999999999,"15min":0.63400000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.73499999999999999,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.60599999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.90800000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81399999999999995,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58599999999999997}]},{"osd":8,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71199999999999997,"15min":0.72799999999999998},"min":{"1min":0.51100000000000001,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.76900000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.747,"15min":0.76400000000000001},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.83899999999999997,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.64400000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.66500000000000004,"15min":0.67000000000000004},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.82999999999999996,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.66600000000000004,"15min":0.66800000000000004},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65100000000000002}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]}]},{"osd":8,"up_from":56,"seq":240518168612,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27936,"kb_used_data":916,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717216,"statfs":{"total":95995035648,"available":95966429184,"internally_reserved":0,"allocated":937984,"data_stored":836487,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54900000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.755,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.379},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76300000000000001,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.432}]},{"osd":2,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.64700000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77700000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79900000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74399999999999999,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80900000000000005,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.498},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.5,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71399999999999997,"15min":0.71599999999999997},"min":{"1min":0.504,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85899999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.56200000000000006,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.88,"5min":1.028,"15min":1.028},"last":0.51200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":0.71999999999999997,"15min":0.70999999999999996},"min":{"1min":0.628,"5min":0.501,"15min":0.501},"max":{"1min":1,"5min":1,"15min":1},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.69499999999999995,"15min":0.68400000000000005},"min":{"1min":0.53700000000000003,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.54000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.70099999999999996,"15min":0.69299999999999995},"min":{"1min":0.59499999999999997,"5min":0.442,"15min":0.442},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.73699999999999999,"15min":0.73299999999999998},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.52600000000000002}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.55400000000000005}]}]},{"osd":7,"up_from":49,"seq":210453397543,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32060,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713092,"statfs":{"total":95995035648,"available":95962206208,"internally_reserved":0,"allocated":966656,"data_stored":1036537,"data_compressed":38638,"data_compressed_allocated":49152,"data_compressed_original":598016,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38900000000000001,"15min":0.373},"min":{"1min":0.313,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58499999999999996,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.42399999999999999,"15min":0.40000000000000002},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.66600000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.314}]},{"osd":2,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.45000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.81899999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.57499999999999996,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73599999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53300000000000003,"15min":0.52200000000000002},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.623},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.54700000000000004},"min":{"1min":0.41499999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.50900000000000001,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.75700000000000001,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.46600000000000003,"15min":0.46300000000000002},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64400000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.59999999999999998,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.623,"15min":0.60599999999999998},"min":{"1min":0.442,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.373}]},{"osd":6,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.63200000000000001,"15min":0.621},"min":{"1min":0.45700000000000002,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.84399999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.58499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78300000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.48699999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.66300000000000003,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81899999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.44500000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.63400000000000001,"15min":0.63100000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80800000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.46500000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85499999999999998,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.47299999999999998}]},{"osd":10,"last update":"Sat Apr 6 14:05:09 2024","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.57199999999999995,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.67700000000000005,"15min":0.67300000000000004},"min":{"1min":0.52100000000000002,"5min":0.438,"15min":0.438},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.498}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.504}]}]},{"osd":6,"up_from":45,"seq":193273528363,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32004,"kb_used_data":888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713148,"statfs":{"total":95995035648,"available":95962263552,"internally_reserved":0,"allocated":909312,"data_stored":443653,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40699999999999997,"15min":0.41099999999999998},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40000000000000002},"min":{"1min":0.28899999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.52400000000000002,"15min":0.50600000000000001},"min":{"1min":0.436,"5min":0.317,"15min":0.317},"max":{"1min":0.74299999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.54100000000000004,"15min":0.52700000000000002},"min":{"1min":0.42299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.446,"5min":1.446,"15min":1.446},"last":0.44},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.57299999999999995,"15min":0.59099999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70399999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.627},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.63},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.624,"15min":0.63400000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.78400000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Apr 6 14:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.65700000000000003,"15min":0.65500000000000003},"min":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.53600000000000003}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.495,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.79900000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.66100000000000003,"15min":0.65400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.497}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.63200000000000001,"15min":0.64400000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.68700000000000006,"15min":0.68100000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.1379999999999999,"5min":4.1379999999999999,"15min":4.1379999999999999},"last":0.64300000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.66900000000000004,"15min":0.65900000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.68799999999999994,"15min":0.68400000000000005},"min":{"1min":0.56100000000000005,"5min":0.503,"15min":0.503},"max":{"1min":1.6499999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.58799999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27968,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717184,"statfs":{"total":95995035648,"available":95966396416,"internally_reserved":0,"allocated":974848,"data_stored":511168,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.35899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.48799999999999999,"15min":0.47299999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.56000000000000005,"15min":0.51000000000000001},"min":{"1min":0.46000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81499999999999995,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.55500000000000005,"15min":0.51700000000000002},"min":{"1min":0.47599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77500000000000002,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52600000000000002,"15min":0.49299999999999999},"min":{"1min":0.38900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.51300000000000001},"min":{"1min":0.46800000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69399999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46100000000000002,"15min":0.44700000000000001},"min":{"1min":0.38500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.41799999999999998},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.59599999999999997,"15min":0.57699999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.871,"5min":0.877,"15min":0.877},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.59099999999999997,"15min":0.56499999999999995},"min":{"1min":0.441,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.79900000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.60299999999999998,"15min":0.58999999999999997},"min":{"1min":0.48699999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81899999999999995,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.47299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.86299999999999999,"5min":0.874,"15min":0.874},"last":0.66600000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.57199999999999995,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64000000000000001,"15min":0.62},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.875,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.64100000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.70099999999999996,"15min":0.69499999999999995},"min":{"1min":0.53400000000000003,"5min":0.503,"15min":0.503},"max":{"1min":0.90700000000000003,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.65700000000000003,"15min":0.65000000000000002},"min":{"1min":0.56499999999999995,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.83799999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.60699999999999998}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.68000000000000005,"15min":0.67600000000000005},"min":{"1min":0.5,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.86399999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.624},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.748,"15min":0.74299999999999999},"min":{"1min":0.56999999999999995,"5min":0.502,"15min":0.502},"max":{"1min":0.91400000000000003,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.70599999999999996}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.71399999999999997,"15min":0.71099999999999997},"min":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.88,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.72299999999999998,"15min":0.71699999999999997},"min":{"1min":0.56799999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.876,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.67600000000000005}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.65200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27924,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717228,"statfs":{"total":95995035648,"available":95966441472,"internally_reserved":0,"allocated":929792,"data_stored":444924,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.49299999999999999,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66400000000000003,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.498,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.624,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57099999999999995,"15min":0.56699999999999995},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.56899999999999995},"min":{"1min":0.42499999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69199999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.435,"5min":0.435,"15min":0.435},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.497,"15min":0.499},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65100000000000002,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.375},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60899999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32600000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.45400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78700000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.44900000000000001,"15min":0.46300000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.499,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.46899999999999997,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.55900000000000005,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.502,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.65500000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.58299999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.57999999999999996,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.55100000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.54700000000000004,"15min":0.56100000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.627,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.432}]},{"osd":10,"last update":"Sat Apr 6 14:05:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.68100000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64100000000000001,"15min":0.64900000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.72299999999999998,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.49399999999999999}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":976,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":999424,"data_stored":512863,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.51300000000000001,"15min":0.46600000000000003},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.46899999999999997,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72999999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79300000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56100000000000005,"15min":0.52600000000000002},"min":{"1min":0.45200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79600000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.443}]},{"osd":3,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.58499999999999996,"15min":0.55500000000000005},"min":{"1min":0.45100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.391,"5min":3.391,"15min":3.391},"last":0.5},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56499999999999995,"15min":0.54200000000000004},"min":{"1min":0.46500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.109,"5min":2.109,"15min":2.109},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.45600000000000002,"15min":0.436},"min":{"1min":0.307,"5min":0.216,"15min":0.216},"max":{"1min":0.78400000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.44400000000000001,"15min":0.42199999999999999},"min":{"1min":0.34999999999999998,"5min":0.214,"15min":0.214},"max":{"1min":0.66800000000000004,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60999999999999999,"15min":0.60099999999999998},"min":{"1min":0.39500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.82599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59499999999999997,"15min":0.57999999999999996},"min":{"1min":0.51300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.45200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.71499999999999997,"15min":0.71099999999999997},"min":{"1min":0.57299999999999995,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.90600000000000003,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.66500000000000004,"15min":0.65500000000000003},"min":{"1min":0.46700000000000003,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53400000000000003}]},{"osd":10,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.74199999999999999,"15min":0.72299999999999998},"min":{"1min":0.63100000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.74199999999999999,"15min":0.72699999999999998},"min":{"1min":0.61499999999999999,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.7799999999999998,"5min":2.7799999999999998,"15min":2.7799999999999998},"last":0.57599999999999996}]},{"osd":11,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56000000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36156,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93708996,"statfs":{"total":95995035648,"available":95958011904,"internally_reserved":0,"allocated":970752,"data_stored":968780,"data_compressed":28220,"data_compressed_allocated":36864,"data_compressed_original":524288,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.33700000000000002,"15min":0.33300000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.42899999999999999,"5min":0.629,"15min":0.629},"last":0.314},{"interface":"front","average":{"1min":0.32300000000000001,"5min":0.34200000000000003,"15min":0.33200000000000002},"min":{"1min":0.253,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.41699999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68999999999999995,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51300000000000001,"15min":0.49099999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70599999999999996,"5min":0.872,"15min":0.872},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.66200000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.436,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.63500000000000001,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.82099999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65200000000000002,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.45500000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.72699999999999998,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.64300000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.82899999999999996}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.47499999999999998,"15min":0.48499999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.55600000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.47599999999999998,"15min":0.49399999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.54300000000000004,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56100000000000005,"15min":0.56200000000000006},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.754,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.754},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76400000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.76400000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:04:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.60199999999999998,"15min":0.626},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.53800000000000003,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.50700000000000001,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.504,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.81000000000000005}]},{"osd":10,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.624,"15min":0.63600000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.72199999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.72999999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.79800000000000004}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.83799999999999997}]}]},{"osd":4,"up_from":33,"seq":141733920819,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717236,"statfs":{"total":95995035648,"available":95966449664,"internally_reserved":0,"allocated":917504,"data_stored":445472,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35399999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.40200000000000002,"15min":0.378},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.70199999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":1.036,"15min":1.036},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.55800000000000005,"15min":0.53600000000000003},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.50700000000000001},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76500000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.441},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.47999999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68500000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.57199999999999995,"15min":0.55400000000000005},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82299999999999995,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.60399999999999998,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79600000000000004,"5min":1.012,"15min":1.012},"last":0.496}]},{"osd":6,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56599999999999995,"15min":0.54200000000000004},"min":{"1min":0.45000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.72599999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54500000000000004,"15min":0.51500000000000001},"min":{"1min":0.45900000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.44900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57699999999999996,"15min":0.56299999999999994},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.35699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.68000000000000005,"15min":0.67700000000000005},"min":{"1min":0.52200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85799999999999998,"5min":0.879,"15min":0.879},"last":0.432},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73899999999999999,"15min":0.746},"min":{"1min":0.57999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81999999999999995,"5min":1.694,"15min":1.694},"last":0.45400000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.71599999999999997,"15min":0.71999999999999997},"min":{"1min":0.53300000000000003,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83299999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.62},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.69599999999999995,"15min":0.69199999999999995},"min":{"1min":0.54500000000000004,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.80400000000000005,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61099999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.76900000000000002,"15min":0.77800000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.86199999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.73199999999999998,"15min":0.73399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.48199999999999998}]},{"osd":11,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.53500000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":1003520,"data_stored":1167352,"data_compressed":44794,"data_compressed_allocated":57344,"data_compressed_original":720896,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.5,"15min":0.48299999999999998},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.70199999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.495,"15min":0.47899999999999998},"min":{"1min":0.36799999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73099999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.53500000000000003,"15min":0.505},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.52500000000000002,"15min":0.497},"min":{"1min":0.44600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.40200000000000002,"15min":0.376},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.65100000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.318},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.41799999999999998,"15min":0.38900000000000001},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":0.65700000000000003,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52500000000000002,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67600000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.54300000000000004,"15min":0.52100000000000002},"min":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.74299999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59999999999999998,"15min":0.57099999999999995},"min":{"1min":0.438,"5min":0.432,"15min":0.432},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.58099999999999996,"15min":0.57099999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59699999999999998,"15min":0.59199999999999997},"min":{"1min":0.45400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80300000000000005,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.629,"15min":0.61899999999999999},"min":{"1min":0.497,"5min":0.378,"15min":0.378},"max":{"1min":0.76400000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.435}]},{"osd":7,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.64600000000000002,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.86199999999999999,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.64100000000000001,"15min":0.623},"min":{"1min":0.48899999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.85399999999999998,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.52600000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.625,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.68400000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.88200000000000001,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.48399999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.69499999999999995,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.84399999999999997,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.753,"15min":0.75800000000000001},"min":{"1min":0.54000000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.94699999999999995,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.54900000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.60799999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.90600000000000003,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.71299999999999997,"15min":0.70699999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":67722,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":69417,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":525100,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":133421,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":393372,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394034,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":201700,"data_compressed":7989,"data_compressed_allocated":12288,"data_compressed_original":131072,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:05:28.762 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3320543710' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/776667042' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1226175357' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:28 smithi134 ceph-mon[47549]: mgrmap e23: x(active, since 4s), standbys: z, y 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 -- 172.21.15.90:0/890256535 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe3440ffae0 msgr2=0x7fe3440fff60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 --2- 172.21.15.90:0/890256535 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe3440ffae0 0x7fe3440fff60 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 crypto rx=0x7fe338009990 tx=0x7fe338009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 -- 172.21.15.90:0/890256535 shutdown_connections 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 --2- 172.21.15.90:0/890256535 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe34410c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 --2- 172.21.15.90:0/890256535 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe344105b80 0x7fe344105f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 --2- 172.21.15.90:0/890256535 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe3440ffae0 0x7fe3440fff60 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 -- 172.21.15.90:0/890256535 >> 172.21.15.90:0/890256535 conn(0x7fe3440fb5f0 msgr2=0x7fe3440fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 -- 172.21.15.90:0/890256535 shutdown_connections 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.917+0000 7fe34a62a700 1 -- 172.21.15.90:0/890256535 wait complete. 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 Processor -- start 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 -- start start 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 0x7fe344197e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe344198340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe344198c10 con 0x7fe344105b80 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3441a33c0 con 0x7fe3441004a0 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe34a62a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3441a3500 con 0x7fe3440ffae0 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe348bc7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe348bc7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50306/0 (socket says 172.21.15.90:50306) 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.918+0000 7fe348bc7700 1 -- 172.21.15.90:0/2596959614 learned_addr learned my addr 172.21.15.90:0/2596959614 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe3437fe700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe344198340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe343fff700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 0x7fe344197e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 -- 172.21.15.90:0/2596959614 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 msgr2=0x7fe344197e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 0x7fe344197e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 -- 172.21.15.90:0/2596959614 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 msgr2=0x7fe344198340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe344198340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 -- 172.21.15.90:0/2596959614 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3380095f0 con 0x7fe344105b80 2024-04-06T14:05:28.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe343fff700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 0x7fe344197e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe3437fe700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe344198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe348bc7700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7fe33c00ef60 tx=0x7fe33c00ef90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe33c018070 con 0x7fe344105b80 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe33c00f040 con 0x7fe344105b80 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe33c003e30 con 0x7fe344105b80 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3441a3780 con 0x7fe344105b80 2024-04-06T14:05:28.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.919+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3441a3c40 con 0x7fe344105b80 2024-04-06T14:05:28.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.920+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe344108100 con 0x7fe344105b80 2024-04-06T14:05:28.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.921+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fe33c004ad0 con 0x7fe344105b80 2024-04-06T14:05:28.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.922+0000 7fe3417fa700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 0x7fe32c0a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:28.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.922+0000 7fe343fff700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 0x7fe32c0a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:28.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.922+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe33c0cd5a0 con 0x7fe344105b80 2024-04-06T14:05:28.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.922+0000 7fe343fff700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 0x7fe32c0a3870 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fe33400baa0 tx=0x7fe334005db0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:28.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:28.925+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe33c096800 con 0x7fe344105b80 2024-04-06T14:05:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: Updating smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: Updating smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: Updating smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: pgmap v5: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3320543710' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/776667042' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1226175357' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:28 smithi105 ceph-mon[46969]: mgrmap e23: x(active, since 4s), standbys: z, y 2024-04-06T14:05:29.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.122+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fe344068410 con 0x7fe32c0a13b0 2024-04-06T14:05:29.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.132+0000 7fe3417fa700 1 -- 172.21.15.90:0/2596959614 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+344496 (secure 0 0 0) 0x7fe344068410 con 0x7fe32c0a13b0 2024-04-06T14:05:29.137 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 msgr2=0x7fe32c0a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 0x7fe32c0a3870 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fe33400baa0 tx=0x7fe334005db0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 msgr2=0x7fe3441a2e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 crypto rx=0x7fe33c00ef60 tx=0x7fe33c00ef90 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 shutdown_connections 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe32c0a13b0 0x7fe32c0a3870 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe344105b80 0x7fe3441a2e80 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe3440ffae0 0x7fe344197e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 --2- 172.21.15.90:0/2596959614 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe3441004a0 0x7fe344198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 >> 172.21.15.90:0/2596959614 conn(0x7fe3440fb5f0 msgr2=0x7fe3440fbbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:29.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 shutdown_connections 2024-04-06T14:05:29.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.136+0000 7fe34a62a700 1 -- 172.21.15.90:0/2596959614 wait complete. 2024-04-06T14:05:29.141 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:05:29.148 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":5,"stamp":"2024-04-06T14:05:27.571371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1187895,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":727,"num_read_kb":518,"num_write":452,"num_write_kb":4368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1181318,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":854,"ondisk_log_size":854,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":360068,"kb_used_data":11268,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124581756,"statfs":{"total":1151940427776,"available":1151571718144,"internally_reserved":0,"allocated":11538432,"data_stored":8884044,"data_compressed":223581,"data_compressed_allocated":294912,"data_compressed_original":3440640,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.988099"},"pg_stats":[{"pgid":"3.3f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615419+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:05:23.615419+0000","last_peered":"2024-04-06T14:05:23.615419+0000","last_clean":"2024-04-06T14:05:23.615419+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:05:23.615419+0000","last_undegraded":"2024-04-06T14:05:23.615419+0000","last_fullsized":"2024-04-06T14:05:23.615419+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607957+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:05:23.607957+0000","last_peered":"2024-04-06T14:05:23.607957+0000","last_clean":"2024-04-06T14:05:23.607957+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:05:23.607957+0000","last_undegraded":"2024-04-06T14:05:23.607957+0000","last_fullsized":"2024-04-06T14:05:23.607957+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616328+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:05:23.616328+0000","last_peered":"2024-04-06T14:05:23.616328+0000","last_clean":"2024-04-06T14:05:23.616328+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:05:23.616328+0000","last_undegraded":"2024-04-06T14:05:23.616328+0000","last_fullsized":"2024-04-06T14:05:23.616328+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.593301+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:05:23.593301+0000","last_peered":"2024-04-06T14:05:23.593301+0000","last_clean":"2024-04-06T14:05:23.593301+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:05:23.593301+0000","last_undegraded":"2024-04-06T14:05:23.593301+0000","last_fullsized":"2024-04-06T14:05:23.593301+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.592862+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:05:23.592862+0000","last_peered":"2024-04-06T14:05:23.592862+0000","last_clean":"2024-04-06T14:05:23.592862+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:05:23.592862+0000","last_undegraded":"2024-04-06T14:05:23.592862+0000","last_fullsized":"2024-04-06T14:05:23.592862+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604597+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:05:23.604597+0000","last_peered":"2024-04-06T14:05:23.604597+0000","last_clean":"2024-04-06T14:05:23.604597+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:05:23.604597+0000","last_undegraded":"2024-04-06T14:05:23.604597+0000","last_fullsized":"2024-04-06T14:05:23.604597+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609927+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:05:23.609927+0000","last_peered":"2024-04-06T14:05:23.609927+0000","last_clean":"2024-04-06T14:05:23.609927+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:05:23.609927+0000","last_undegraded":"2024-04-06T14:05:23.609927+0000","last_fullsized":"2024-04-06T14:05:23.609927+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603861+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:05:23.603861+0000","last_peered":"2024-04-06T14:05:23.603861+0000","last_clean":"2024-04-06T14:05:23.603861+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:05:23.603861+0000","last_undegraded":"2024-04-06T14:05:23.603861+0000","last_fullsized":"2024-04-06T14:05:23.603861+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603421+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:05:23.603421+0000","last_peered":"2024-04-06T14:05:23.603421+0000","last_clean":"2024-04-06T14:05:23.603421+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:05:23.603421+0000","last_undegraded":"2024-04-06T14:05:23.603421+0000","last_fullsized":"2024-04-06T14:05:23.603421+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"84'16","reported_seq":30,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603913+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:05:23.603913+0000","last_peered":"2024-04-06T14:05:23.603913+0000","last_clean":"2024-04-06T14:05:23.603913+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:05:23.603913+0000","last_undegraded":"2024-04-06T14:05:23.603913+0000","last_fullsized":"2024-04-06T14:05:23.603913+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609304+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:05:23.609304+0000","last_peered":"2024-04-06T14:05:23.609304+0000","last_clean":"2024-04-06T14:05:23.609304+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:05:23.609304+0000","last_undegraded":"2024-04-06T14:05:23.609304+0000","last_fullsized":"2024-04-06T14:05:23.609304+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602535+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:05:23.602535+0000","last_peered":"2024-04-06T14:05:23.602535+0000","last_clean":"2024-04-06T14:05:23.602535+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:05:23.602535+0000","last_undegraded":"2024-04-06T14:05:23.602535+0000","last_fullsized":"2024-04-06T14:05:23.602535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610171+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:05:23.610171+0000","last_peered":"2024-04-06T14:05:23.610171+0000","last_clean":"2024-04-06T14:05:23.610171+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:05:23.610171+0000","last_undegraded":"2024-04-06T14:05:23.610171+0000","last_fullsized":"2024-04-06T14:05:23.610171+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611310+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:05:23.611310+0000","last_peered":"2024-04-06T14:05:23.611310+0000","last_clean":"2024-04-06T14:05:23.611310+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:05:23.611310+0000","last_undegraded":"2024-04-06T14:05:23.611310+0000","last_fullsized":"2024-04-06T14:05:23.611310+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614994+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:05:23.614994+0000","last_peered":"2024-04-06T14:05:23.614994+0000","last_clean":"2024-04-06T14:05:23.614994+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:05:23.614994+0000","last_undegraded":"2024-04-06T14:05:23.614994+0000","last_fullsized":"2024-04-06T14:05:23.614994+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611373+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:05:23.611373+0000","last_peered":"2024-04-06T14:05:23.611373+0000","last_clean":"2024-04-06T14:05:23.611373+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:05:23.611373+0000","last_undegraded":"2024-04-06T14:05:23.611373+0000","last_fullsized":"2024-04-06T14:05:23.611373+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615524+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:05:23.615524+0000","last_peered":"2024-04-06T14:05:23.615524+0000","last_clean":"2024-04-06T14:05:23.615524+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:05:23.615524+0000","last_undegraded":"2024-04-06T14:05:23.615524+0000","last_fullsized":"2024-04-06T14:05:23.615524+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613532+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:05:23.613532+0000","last_peered":"2024-04-06T14:05:23.613532+0000","last_clean":"2024-04-06T14:05:23.613532+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:05:23.613532+0000","last_undegraded":"2024-04-06T14:05:23.613532+0000","last_fullsized":"2024-04-06T14:05:23.613532+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611801+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:05:23.611801+0000","last_peered":"2024-04-06T14:05:23.611801+0000","last_clean":"2024-04-06T14:05:23.611801+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:05:23.611801+0000","last_undegraded":"2024-04-06T14:05:23.611801+0000","last_fullsized":"2024-04-06T14:05:23.611801+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":44,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.235666+0000","last_change":"2024-04-06T14:05:05.235370+0000","last_active":"2024-04-06T14:05:05.235666+0000","last_peered":"2024-04-06T14:05:05.235666+0000","last_clean":"2024-04-06T14:05:05.235666+0000","last_became_active":"2024-04-06T14:05:05.229253+0000","last_became_peered":"2024-04-06T14:05:05.229253+0000","last_unstale":"2024-04-06T14:05:05.235666+0000","last_undegraded":"2024-04-06T14:05:05.235666+0000","last_fullsized":"2024-04-06T14:05:05.235666+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:37:23.805018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,4],"acting":[9,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596468+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:05:23.596468+0000","last_peered":"2024-04-06T14:05:23.596468+0000","last_clean":"2024-04-06T14:05:23.596468+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:05:23.596468+0000","last_undegraded":"2024-04-06T14:05:23.596468+0000","last_fullsized":"2024-04-06T14:05:23.596468+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190339+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:05:05.190339+0000","last_peered":"2024-04-06T14:05:05.190339+0000","last_clean":"2024-04-06T14:05:05.190339+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:05:05.190339+0000","last_undegraded":"2024-04-06T14:05:05.190339+0000","last_fullsized":"2024-04-06T14:05:05.190339+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617749+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:05:23.617749+0000","last_peered":"2024-04-06T14:05:23.617749+0000","last_clean":"2024-04-06T14:05:23.617749+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:05:23.617749+0000","last_undegraded":"2024-04-06T14:05:23.617749+0000","last_fullsized":"2024-04-06T14:05:23.617749+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196231+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:05:05.196231+0000","last_peered":"2024-04-06T14:05:05.196231+0000","last_clean":"2024-04-06T14:05:05.196231+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:05:05.196231+0000","last_undegraded":"2024-04-06T14:05:05.196231+0000","last_fullsized":"2024-04-06T14:05:05.196231+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607848+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:05:23.607848+0000","last_peered":"2024-04-06T14:05:23.607848+0000","last_clean":"2024-04-06T14:05:23.607848+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:05:23.607848+0000","last_undegraded":"2024-04-06T14:05:23.607848+0000","last_fullsized":"2024-04-06T14:05:23.607848+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195822+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:05:05.195822+0000","last_peered":"2024-04-06T14:05:05.195822+0000","last_clean":"2024-04-06T14:05:05.195822+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:05:05.195822+0000","last_undegraded":"2024-04-06T14:05:05.195822+0000","last_fullsized":"2024-04-06T14:05:05.195822+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"84'15","reported_seq":38,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600996+0000","last_change":"2024-04-06T14:05:05.196872+0000","last_active":"2024-04-06T14:05:23.600996+0000","last_peered":"2024-04-06T14:05:23.600996+0000","last_clean":"2024-04-06T14:05:23.600996+0000","last_became_active":"2024-04-06T14:05:05.196522+0000","last_became_peered":"2024-04-06T14:05:05.196522+0000","last_unstale":"2024-04-06T14:05:23.600996+0000","last_undegraded":"2024-04-06T14:05:23.600996+0000","last_fullsized":"2024-04-06T14:05:23.600996+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:12:54.386682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,1],"acting":[5,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"84'6","reported_seq":101,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626416+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:05:23.626416+0000","last_peered":"2024-04-06T14:05:23.626416+0000","last_clean":"2024-04-06T14:05:23.626416+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:05:23.626416+0000","last_undegraded":"2024-04-06T14:05:23.626416+0000","last_fullsized":"2024-04-06T14:05:23.626416+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":967,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607842+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:05:23.607842+0000","last_peered":"2024-04-06T14:05:23.607842+0000","last_clean":"2024-04-06T14:05:23.607842+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:05:23.607842+0000","last_undegraded":"2024-04-06T14:05:23.607842+0000","last_fullsized":"2024-04-06T14:05:23.607842+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.259157+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:05:05.259157+0000","last_peered":"2024-04-06T14:05:05.259157+0000","last_clean":"2024-04-06T14:05:05.259157+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:05:05.259157+0000","last_undegraded":"2024-04-06T14:05:05.259157+0000","last_fullsized":"2024-04-06T14:05:05.259157+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.599735+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:05:23.599735+0000","last_peered":"2024-04-06T14:05:23.599735+0000","last_clean":"2024-04-06T14:05:23.599735+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:05:23.599735+0000","last_undegraded":"2024-04-06T14:05:23.599735+0000","last_fullsized":"2024-04-06T14:05:23.599735+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193183+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:05:05.193183+0000","last_peered":"2024-04-06T14:05:05.193183+0000","last_clean":"2024-04-06T14:05:05.193183+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:05:05.193183+0000","last_undegraded":"2024-04-06T14:05:05.193183+0000","last_fullsized":"2024-04-06T14:05:05.193183+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604741+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:05:23.604741+0000","last_peered":"2024-04-06T14:05:23.604741+0000","last_clean":"2024-04-06T14:05:23.604741+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:05:23.604741+0000","last_undegraded":"2024-04-06T14:05:23.604741+0000","last_fullsized":"2024-04-06T14:05:23.604741+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189600+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:05:05.189600+0000","last_peered":"2024-04-06T14:05:05.189600+0000","last_clean":"2024-04-06T14:05:05.189600+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:05:05.189600+0000","last_undegraded":"2024-04-06T14:05:05.189600+0000","last_fullsized":"2024-04-06T14:05:05.189600+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607641+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:05:23.607641+0000","last_peered":"2024-04-06T14:05:23.607641+0000","last_clean":"2024-04-06T14:05:23.607641+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:05:23.607641+0000","last_undegraded":"2024-04-06T14:05:23.607641+0000","last_fullsized":"2024-04-06T14:05:23.607641+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"84'4","reported_seq":97,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626088+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:05:23.626088+0000","last_peered":"2024-04-06T14:05:23.626088+0000","last_clean":"2024-04-06T14:05:23.626088+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:05:23.626088+0000","last_undegraded":"2024-04-06T14:05:23.626088+0000","last_fullsized":"2024-04-06T14:05:23.626088+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614466+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:05:23.614466+0000","last_peered":"2024-04-06T14:05:23.614466+0000","last_clean":"2024-04-06T14:05:23.614466+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:05:23.614466+0000","last_undegraded":"2024-04-06T14:05:23.614466+0000","last_fullsized":"2024-04-06T14:05:23.614466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194821+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:05:05.194821+0000","last_peered":"2024-04-06T14:05:05.194821+0000","last_clean":"2024-04-06T14:05:05.194821+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:05:05.194821+0000","last_undegraded":"2024-04-06T14:05:05.194821+0000","last_fullsized":"2024-04-06T14:05:05.194821+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610283+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:05:23.610283+0000","last_peered":"2024-04-06T14:05:23.610283+0000","last_clean":"2024-04-06T14:05:23.610283+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:05:23.610283+0000","last_undegraded":"2024-04-06T14:05:23.610283+0000","last_fullsized":"2024-04-06T14:05:23.610283+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190092+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:05:05.190092+0000","last_peered":"2024-04-06T14:05:05.190092+0000","last_clean":"2024-04-06T14:05:05.190092+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:05:05.190092+0000","last_undegraded":"2024-04-06T14:05:05.190092+0000","last_fullsized":"2024-04-06T14:05:05.190092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604056+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:05:23.604056+0000","last_peered":"2024-04-06T14:05:23.604056+0000","last_clean":"2024-04-06T14:05:23.604056+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:05:23.604056+0000","last_undegraded":"2024-04-06T14:05:23.604056+0000","last_fullsized":"2024-04-06T14:05:23.604056+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"84'34","reported_seq":215,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.840919+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:05:23.840919+0000","last_peered":"2024-04-06T14:05:23.840919+0000","last_clean":"2024-04-06T14:05:23.840919+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:05:23.840919+0000","last_undegraded":"2024-04-06T14:05:23.840919+0000","last_fullsized":"2024-04-06T14:05:23.840919+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67722,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":166,"num_read_kb":143,"num_write":35,"num_write_kb":80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605924+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:05:23.605924+0000","last_peered":"2024-04-06T14:05:23.605924+0000","last_clean":"2024-04-06T14:05:23.605924+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:05:23.605924+0000","last_undegraded":"2024-04-06T14:05:23.605924+0000","last_fullsized":"2024-04-06T14:05:23.605924+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":22,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191794+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:05:05.191794+0000","last_peered":"2024-04-06T14:05:05.191794+0000","last_clean":"2024-04-06T14:05:05.191794+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:05:05.191794+0000","last_undegraded":"2024-04-06T14:05:05.191794+0000","last_fullsized":"2024-04-06T14:05:05.191794+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612189+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:05:23.612189+0000","last_peered":"2024-04-06T14:05:23.612189+0000","last_clean":"2024-04-06T14:05:23.612189+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:05:23.612189+0000","last_undegraded":"2024-04-06T14:05:23.612189+0000","last_fullsized":"2024-04-06T14:05:23.612189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189874+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:05:05.189874+0000","last_peered":"2024-04-06T14:05:05.189874+0000","last_clean":"2024-04-06T14:05:05.189874+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:05:05.189874+0000","last_undegraded":"2024-04-06T14:05:05.189874+0000","last_fullsized":"2024-04-06T14:05:05.189874+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610716+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:05:23.610716+0000","last_peered":"2024-04-06T14:05:23.610716+0000","last_clean":"2024-04-06T14:05:23.610716+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:05:23.610716+0000","last_undegraded":"2024-04-06T14:05:23.610716+0000","last_fullsized":"2024-04-06T14:05:23.610716+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"84'6","reported_seq":104,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.629802+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:05:23.629802+0000","last_peered":"2024-04-06T14:05:23.629802+0000","last_clean":"2024-04-06T14:05:23.629802+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:05:23.629802+0000","last_undegraded":"2024-04-06T14:05:23.629802+0000","last_fullsized":"2024-04-06T14:05:23.629802+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1388,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":84,"num_read_kb":6,"num_write":5,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605694+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:05:23.605694+0000","last_peered":"2024-04-06T14:05:23.605694+0000","last_clean":"2024-04-06T14:05:23.605694+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:05:23.605694+0000","last_undegraded":"2024-04-06T14:05:23.605694+0000","last_fullsized":"2024-04-06T14:05:23.605694+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"81'15","reported_seq":31,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195316+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:05:05.195316+0000","last_peered":"2024-04-06T14:05:05.195316+0000","last_clean":"2024-04-06T14:05:05.195316+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:05:05.195316+0000","last_undegraded":"2024-04-06T14:05:05.195316+0000","last_fullsized":"2024-04-06T14:05:05.195316+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602036+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:05:23.602036+0000","last_peered":"2024-04-06T14:05:23.602036+0000","last_clean":"2024-04-06T14:05:23.602036+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:05:23.602036+0000","last_undegraded":"2024-04-06T14:05:23.602036+0000","last_fullsized":"2024-04-06T14:05:23.602036+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'2","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196332+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:05:05.196332+0000","last_peered":"2024-04-06T14:05:05.196332+0000","last_clean":"2024-04-06T14:05:05.196332+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:05:05.196332+0000","last_undegraded":"2024-04-06T14:05:05.196332+0000","last_fullsized":"2024-04-06T14:05:05.196332+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604329+0000","last_change":"2024-04-06T14:05:05.196327+0000","last_active":"2024-04-06T14:05:23.604329+0000","last_peered":"2024-04-06T14:05:23.604329+0000","last_clean":"2024-04-06T14:05:23.604329+0000","last_became_active":"2024-04-06T14:05:05.195845+0000","last_became_peered":"2024-04-06T14:05:05.195845+0000","last_unstale":"2024-04-06T14:05:23.604329+0000","last_undegraded":"2024-04-06T14:05:23.604329+0000","last_fullsized":"2024-04-06T14:05:23.604329+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:16:06.262603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,6],"acting":[11,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191665+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:05:05.191665+0000","last_peered":"2024-04-06T14:05:05.191665+0000","last_clean":"2024-04-06T14:05:05.191665+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:05:05.191665+0000","last_undegraded":"2024-04-06T14:05:05.191665+0000","last_fullsized":"2024-04-06T14:05:05.191665+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.597349+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:05:23.597349+0000","last_peered":"2024-04-06T14:05:23.597349+0000","last_clean":"2024-04-06T14:05:23.597349+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:05:23.597349+0000","last_undegraded":"2024-04-06T14:05:23.597349+0000","last_fullsized":"2024-04-06T14:05:23.597349+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194517+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:05:05.194517+0000","last_peered":"2024-04-06T14:05:05.194517+0000","last_clean":"2024-04-06T14:05:05.194517+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:05:05.194517+0000","last_undegraded":"2024-04-06T14:05:05.194517+0000","last_fullsized":"2024-04-06T14:05:05.194517+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613134+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:05:23.613134+0000","last_peered":"2024-04-06T14:05:23.613134+0000","last_clean":"2024-04-06T14:05:23.613134+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:05:23.613134+0000","last_undegraded":"2024-04-06T14:05:23.613134+0000","last_fullsized":"2024-04-06T14:05:23.613134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604058+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:05:23.604058+0000","last_peered":"2024-04-06T14:05:23.604058+0000","last_clean":"2024-04-06T14:05:23.604058+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:05:23.604058+0000","last_undegraded":"2024-04-06T14:05:23.604058+0000","last_fullsized":"2024-04-06T14:05:23.604058+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"83'5","reported_seq":98,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.576804+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:05:23.576804+0000","last_peered":"2024-04-06T14:05:23.576804+0000","last_clean":"2024-04-06T14:05:23.576804+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:05:23.576804+0000","last_undegraded":"2024-04-06T14:05:23.576804+0000","last_fullsized":"2024-04-06T14:05:23.576804+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604005+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:05:23.604005+0000","last_peered":"2024-04-06T14:05:23.604005+0000","last_clean":"2024-04-06T14:05:23.604005+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:05:23.604005+0000","last_undegraded":"2024-04-06T14:05:23.604005+0000","last_fullsized":"2024-04-06T14:05:23.604005+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228303+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:05:05.228303+0000","last_peered":"2024-04-06T14:05:05.228303+0000","last_clean":"2024-04-06T14:05:05.228303+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:05:05.228303+0000","last_undegraded":"2024-04-06T14:05:05.228303+0000","last_fullsized":"2024-04-06T14:05:05.228303+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":337,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194265+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:05:05.194265+0000","last_peered":"2024-04-06T14:05:05.194265+0000","last_clean":"2024-04-06T14:05:05.194265+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:05:05.194265+0000","last_undegraded":"2024-04-06T14:05:05.194265+0000","last_fullsized":"2024-04-06T14:05:05.194265+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610576+0000","last_change":"2024-04-06T14:05:05.195151+0000","last_active":"2024-04-06T14:05:23.610576+0000","last_peered":"2024-04-06T14:05:23.610576+0000","last_clean":"2024-04-06T14:05:23.610576+0000","last_became_active":"2024-04-06T14:05:05.194828+0000","last_became_peered":"2024-04-06T14:05:05.194828+0000","last_unstale":"2024-04-06T14:05:23.610576+0000","last_undegraded":"2024-04-06T14:05:23.610576+0000","last_fullsized":"2024-04-06T14:05:23.610576+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:11:32.067936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,2],"acting":[6,9,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194363+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:05:05.194363+0000","last_peered":"2024-04-06T14:05:05.194363+0000","last_clean":"2024-04-06T14:05:05.194363+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:05:05.194363+0000","last_undegraded":"2024-04-06T14:05:05.194363+0000","last_fullsized":"2024-04-06T14:05:05.194363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"81'1","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196094+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:05:05.196094+0000","last_peered":"2024-04-06T14:05:05.196094+0000","last_clean":"2024-04-06T14:05:05.196094+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:05:05.196094+0000","last_undegraded":"2024-04-06T14:05:05.196094+0000","last_fullsized":"2024-04-06T14:05:05.196094+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609695+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:05:23.609695+0000","last_peered":"2024-04-06T14:05:23.609695+0000","last_clean":"2024-04-06T14:05:23.609695+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:05:23.609695+0000","last_undegraded":"2024-04-06T14:05:23.609695+0000","last_fullsized":"2024-04-06T14:05:23.609695+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"84'39","reported_seq":77,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.797513+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:05:23.797513+0000","last_peered":"2024-04-06T14:05:23.797513+0000","last_clean":"2024-04-06T14:05:23.797513+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:05:23.797513+0000","last_undegraded":"2024-04-06T14:05:23.797513+0000","last_fullsized":"2024-04-06T14:05:23.797513+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":68,"num_write_kb":833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"84'5","reported_seq":17,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616610+0000","last_change":"2024-04-06T14:05:05.196110+0000","last_active":"2024-04-06T14:05:23.616610+0000","last_peered":"2024-04-06T14:05:23.616610+0000","last_clean":"2024-04-06T14:05:23.616610+0000","last_became_active":"2024-04-06T14:05:05.195775+0000","last_became_peered":"2024-04-06T14:05:05.195775+0000","last_unstale":"2024-04-06T14:05:23.616610+0000","last_undegraded":"2024-04-06T14:05:23.616610+0000","last_fullsized":"2024-04-06T14:05:23.616610+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:12:33.956229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,7],"acting":[9,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228555+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:05:05.228555+0000","last_peered":"2024-04-06T14:05:05.228555+0000","last_clean":"2024-04-06T14:05:05.228555+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:05:05.228555+0000","last_undegraded":"2024-04-06T14:05:05.228555+0000","last_fullsized":"2024-04-06T14:05:05.228555+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617648+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:05:23.617648+0000","last_peered":"2024-04-06T14:05:23.617648+0000","last_clean":"2024-04-06T14:05:23.617648+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:05:23.617648+0000","last_undegraded":"2024-04-06T14:05:23.617648+0000","last_fullsized":"2024-04-06T14:05:23.617648+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":30,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.200728+0000","last_change":"2024-04-06T14:05:05.200365+0000","last_active":"2024-04-06T14:05:05.200728+0000","last_peered":"2024-04-06T14:05:05.200728+0000","last_clean":"2024-04-06T14:05:05.200728+0000","last_became_active":"2024-04-06T14:05:05.196172+0000","last_became_peered":"2024-04-06T14:05:05.196172+0000","last_unstale":"2024-04-06T14:05:05.200728+0000","last_undegraded":"2024-04-06T14:05:05.200728+0000","last_fullsized":"2024-04-06T14:05:05.200728+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:44.928650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.575363+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:05:23.575363+0000","last_peered":"2024-04-06T14:05:23.575363+0000","last_clean":"2024-04-06T14:05:23.575363+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:05:23.575363+0000","last_undegraded":"2024-04-06T14:05:23.575363+0000","last_fullsized":"2024-04-06T14:05:23.575363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189911+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:05:05.189911+0000","last_peered":"2024-04-06T14:05:05.189911+0000","last_clean":"2024-04-06T14:05:05.189911+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:05:05.189911+0000","last_undegraded":"2024-04-06T14:05:05.189911+0000","last_fullsized":"2024-04-06T14:05:05.189911+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615526+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:05:23.615526+0000","last_peered":"2024-04-06T14:05:23.615526+0000","last_clean":"2024-04-06T14:05:23.615526+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:05:23.615526+0000","last_undegraded":"2024-04-06T14:05:23.615526+0000","last_fullsized":"2024-04-06T14:05:23.615526+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:06.497211+0000","last_change":"2024-04-06T14:05:05.200944+0000","last_active":"2024-04-06T14:05:06.497211+0000","last_peered":"2024-04-06T14:05:06.497211+0000","last_clean":"2024-04-06T14:05:06.497211+0000","last_became_active":"2024-04-06T14:05:05.196972+0000","last_became_peered":"2024-04-06T14:05:05.196972+0000","last_unstale":"2024-04-06T14:05:06.497211+0000","last_undegraded":"2024-04-06T14:05:06.497211+0000","last_fullsized":"2024-04-06T14:05:06.497211+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:00:15.481685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,3,8],"acting":[11,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600338+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:05:23.600338+0000","last_peered":"2024-04-06T14:05:23.600338+0000","last_clean":"2024-04-06T14:05:23.600338+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:05:23.600338+0000","last_undegraded":"2024-04-06T14:05:23.600338+0000","last_fullsized":"2024-04-06T14:05:23.600338+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195568+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:05:05.195568+0000","last_peered":"2024-04-06T14:05:05.195568+0000","last_clean":"2024-04-06T14:05:05.195568+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:05:05.195568+0000","last_undegraded":"2024-04-06T14:05:05.195568+0000","last_fullsized":"2024-04-06T14:05:05.195568+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612506+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:05:23.612506+0000","last_peered":"2024-04-06T14:05:23.612506+0000","last_clean":"2024-04-06T14:05:23.612506+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:05:23.612506+0000","last_undegraded":"2024-04-06T14:05:23.612506+0000","last_fullsized":"2024-04-06T14:05:23.612506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191823+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:05:05.191823+0000","last_peered":"2024-04-06T14:05:05.191823+0000","last_clean":"2024-04-06T14:05:05.191823+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:05:05.191823+0000","last_undegraded":"2024-04-06T14:05:05.191823+0000","last_fullsized":"2024-04-06T14:05:05.191823+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611707+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:05:23.611707+0000","last_peered":"2024-04-06T14:05:23.611707+0000","last_clean":"2024-04-06T14:05:23.611707+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:05:23.611707+0000","last_undegraded":"2024-04-06T14:05:23.611707+0000","last_fullsized":"2024-04-06T14:05:23.611707+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"84'39","reported_seq":62,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.794092+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:05:23.794092+0000","last_peered":"2024-04-06T14:05:23.794092+0000","last_clean":"2024-04-06T14:05:23.794092+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:05:23.794092+0000","last_undegraded":"2024-04-06T14:05:23.794092+0000","last_fullsized":"2024-04-06T14:05:23.794092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606466+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:05:23.606466+0000","last_peered":"2024-04-06T14:05:23.606466+0000","last_clean":"2024-04-06T14:05:23.606466+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:05:23.606466+0000","last_undegraded":"2024-04-06T14:05:23.606466+0000","last_fullsized":"2024-04-06T14:05:23.606466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193472+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:05:05.193472+0000","last_peered":"2024-04-06T14:05:05.193472+0000","last_clean":"2024-04-06T14:05:05.193472+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:05:05.193472+0000","last_undegraded":"2024-04-06T14:05:05.193472+0000","last_fullsized":"2024-04-06T14:05:05.193472+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.608430+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:05:23.608430+0000","last_peered":"2024-04-06T14:05:23.608430+0000","last_clean":"2024-04-06T14:05:23.608430+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:05:23.608430+0000","last_undegraded":"2024-04-06T14:05:23.608430+0000","last_fullsized":"2024-04-06T14:05:23.608430+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"84'9","reported_seq":32,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609331+0000","last_change":"2024-04-06T14:05:05.198436+0000","last_active":"2024-04-06T14:05:23.609331+0000","last_peered":"2024-04-06T14:05:23.609331+0000","last_clean":"2024-04-06T14:05:23.609331+0000","last_became_active":"2024-04-06T14:05:05.198028+0000","last_became_peered":"2024-04-06T14:05:05.198028+0000","last_unstale":"2024-04-06T14:05:23.609331+0000","last_undegraded":"2024-04-06T14:05:23.609331+0000","last_fullsized":"2024-04-06T14:05:23.609331+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:49:16.741498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"84'12","reported_seq":26,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604089+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:05:23.604089+0000","last_peered":"2024-04-06T14:05:23.604089+0000","last_clean":"2024-04-06T14:05:23.604089+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:05:23.604089+0000","last_undegraded":"2024-04-06T14:05:23.604089+0000","last_fullsized":"2024-04-06T14:05:23.604089+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615361+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:05:23.615361+0000","last_peered":"2024-04-06T14:05:23.615361+0000","last_clean":"2024-04-06T14:05:23.615361+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:05:23.615361+0000","last_undegraded":"2024-04-06T14:05:23.615361+0000","last_fullsized":"2024-04-06T14:05:23.615361+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615434+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:05:23.615434+0000","last_peered":"2024-04-06T14:05:23.615434+0000","last_clean":"2024-04-06T14:05:23.615434+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:05:23.615434+0000","last_undegraded":"2024-04-06T14:05:23.615434+0000","last_fullsized":"2024-04-06T14:05:23.615434+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615556+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:05:23.615556+0000","last_peered":"2024-04-06T14:05:23.615556+0000","last_clean":"2024-04-06T14:05:23.615556+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:05:23.615556+0000","last_undegraded":"2024-04-06T14:05:23.615556+0000","last_fullsized":"2024-04-06T14:05:23.615556+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604845+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:05:23.604845+0000","last_peered":"2024-04-06T14:05:23.604845+0000","last_clean":"2024-04-06T14:05:23.604845+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:05:23.604845+0000","last_undegraded":"2024-04-06T14:05:23.604845+0000","last_fullsized":"2024-04-06T14:05:23.604845+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600672+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:05:23.600672+0000","last_peered":"2024-04-06T14:05:23.600672+0000","last_clean":"2024-04-06T14:05:23.600672+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:05:23.600672+0000","last_undegraded":"2024-04-06T14:05:23.600672+0000","last_fullsized":"2024-04-06T14:05:23.600672+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596144+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:05:23.596144+0000","last_peered":"2024-04-06T14:05:23.596144+0000","last_clean":"2024-04-06T14:05:23.596144+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:05:23.596144+0000","last_undegraded":"2024-04-06T14:05:23.596144+0000","last_fullsized":"2024-04-06T14:05:23.596144+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606756+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:05:23.606756+0000","last_peered":"2024-04-06T14:05:23.606756+0000","last_clean":"2024-04-06T14:05:23.606756+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:05:23.606756+0000","last_undegraded":"2024-04-06T14:05:23.606756+0000","last_fullsized":"2024-04-06T14:05:23.606756+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606960+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:05:23.606960+0000","last_peered":"2024-04-06T14:05:23.606960+0000","last_clean":"2024-04-06T14:05:23.606960+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:05:23.606960+0000","last_undegraded":"2024-04-06T14:05:23.606960+0000","last_fullsized":"2024-04-06T14:05:23.606960+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613612+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:05:23.613612+0000","last_peered":"2024-04-06T14:05:23.613612+0000","last_clean":"2024-04-06T14:05:23.613612+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:05:23.613612+0000","last_undegraded":"2024-04-06T14:05:23.613612+0000","last_fullsized":"2024-04-06T14:05:23.613612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615353+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:05:23.615353+0000","last_peered":"2024-04-06T14:05:23.615353+0000","last_clean":"2024-04-06T14:05:23.615353+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:05:23.615353+0000","last_undegraded":"2024-04-06T14:05:23.615353+0000","last_fullsized":"2024-04-06T14:05:23.615353+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.598200+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:05:23.598200+0000","last_peered":"2024-04-06T14:05:23.598200+0000","last_clean":"2024-04-06T14:05:23.598200+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:05:23.598200+0000","last_undegraded":"2024-04-06T14:05:23.598200+0000","last_fullsized":"2024-04-06T14:05:23.598200+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":107160,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597293,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":591,"num_read_kb":399,"num_write":269,"num_write_kb":1428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":565248,"data_stored":1791879,"data_compressed":81405,"data_compressed_allocated":110592,"data_compressed_original":1572864,"omap_allocated":0,"internal_metadata":0},"log_size":215,"ondisk_log_size":215,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514523,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32000,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713152,"statfs":{"total":95995035648,"available":95962267648,"internally_reserved":0,"allocated":905216,"data_stored":444028,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.504},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.745,"15min":0.745},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.58199999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.436},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55700000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.55000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.57499999999999996}]}]},{"osd":10,"up_from":68,"seq":292057776157,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28072,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717080,"statfs":{"total":95995035648,"available":95966289920,"internally_reserved":0,"allocated":1077248,"data_stored":1235300,"data_compressed":46627,"data_compressed_allocated":61440,"data_compressed_original":729088,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.61099999999999999,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.311},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.51600000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.74399999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71399999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":0.65300000000000002},"min":{"1min":0.54600000000000004,"5min":0.501,"15min":0.501},"max":{"1min":0.84099999999999997,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67000000000000004,"15min":0.67300000000000004},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.84999999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.37}]},{"osd":6,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.68000000000000005,"15min":0.68799999999999994},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.75600000000000001,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.74099999999999999,"15min":0.71699999999999997},"min":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.46899999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.76400000000000001,"15min":0.76500000000000001},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.78500000000000003,"15min":0.79000000000000004},"min":{"1min":0.64900000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.92400000000000004,"5min":1.083,"15min":1.083},"last":0.51000000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.76000000000000001,"15min":0.76300000000000001},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":0.878,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76000000000000001,"15min":0.76600000000000001},"min":{"1min":0.58099999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.83199999999999996,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.53000000000000003}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.498},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.54600000000000004}]}]},{"osd":9,"up_from":63,"seq":270582939681,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":946176,"data_stored":837480,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,6,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40799999999999997,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.371},{"interface":"front","average":{"1min":0.371,"5min":0.44900000000000001,"15min":0.46200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.55600000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.314}]},{"osd":1,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.753,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75700000000000001,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.751,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.59599999999999997,"15min":0.60899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.74299999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.49399999999999999,"15min":0.50700000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.624,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69999999999999996,"5min":0.753,"15min":0.753},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88300000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.65600000000000003,"15min":0.67100000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.79400000000000004,"5min":1.107,"15min":1.107},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.63900000000000001,"15min":0.65200000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77800000000000002,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.57499999999999996,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72499999999999998,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.61699999999999999,"15min":0.63400000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.73499999999999999,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.60599999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.90800000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81399999999999995,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58599999999999997}]},{"osd":8,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71199999999999997,"15min":0.72799999999999998},"min":{"1min":0.51100000000000001,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.76900000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.747,"15min":0.76400000000000001},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.83899999999999997,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.64400000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.66500000000000004,"15min":0.67000000000000004},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.82999999999999996,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.66600000000000004,"15min":0.66800000000000004},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65100000000000002}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]}]},{"osd":8,"up_from":56,"seq":240518168612,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27936,"kb_used_data":916,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717216,"statfs":{"total":95995035648,"available":95966429184,"internally_reserved":0,"allocated":937984,"data_stored":836487,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54900000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.755,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.379},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76300000000000001,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.432}]},{"osd":2,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.64700000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77700000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79900000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74399999999999999,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80900000000000005,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.498},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.5,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71399999999999997,"15min":0.71599999999999997},"min":{"1min":0.504,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85899999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.56200000000000006,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.88,"5min":1.028,"15min":1.028},"last":0.51200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":0.71999999999999997,"15min":0.70999999999999996},"min":{"1min":0.628,"5min":0.501,"15min":0.501},"max":{"1min":1,"5min":1,"15min":1},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.69499999999999995,"15min":0.68400000000000005},"min":{"1min":0.53700000000000003,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.54000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.70099999999999996,"15min":0.69299999999999995},"min":{"1min":0.59499999999999997,"5min":0.442,"15min":0.442},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.73699999999999999,"15min":0.73299999999999998},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.52600000000000002}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.55400000000000005}]}]},{"osd":7,"up_from":49,"seq":210453397543,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32060,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713092,"statfs":{"total":95995035648,"available":95962206208,"internally_reserved":0,"allocated":966656,"data_stored":1036537,"data_compressed":38638,"data_compressed_allocated":49152,"data_compressed_original":598016,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38900000000000001,"15min":0.373},"min":{"1min":0.313,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58499999999999996,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.42399999999999999,"15min":0.40000000000000002},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.66600000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.314}]},{"osd":2,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.45000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.81899999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.57499999999999996,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73599999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53300000000000003,"15min":0.52200000000000002},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.623},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.54700000000000004},"min":{"1min":0.41499999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.50900000000000001,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.75700000000000001,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.46600000000000003,"15min":0.46300000000000002},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64400000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.59999999999999998,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.623,"15min":0.60599999999999998},"min":{"1min":0.442,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.373}]},{"osd":6,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.63200000000000001,"15min":0.621},"min":{"1min":0.45700000000000002,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.84399999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.58499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78300000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.48699999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.66300000000000003,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81899999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.44500000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.63400000000000001,"15min":0.63100000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80800000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.46500000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85499999999999998,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.47299999999999998}]},{"osd":10,"last update":"Sat Apr 6 14:05:09 2024","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.57199999999999995,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.67700000000000005,"15min":0.67300000000000004},"min":{"1min":0.52100000000000002,"5min":0.438,"15min":0.438},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.498}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.504}]}]},{"osd":6,"up_from":45,"seq":193273528363,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32004,"kb_used_data":888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713148,"statfs":{"total":95995035648,"available":95962263552,"internally_reserved":0,"allocated":909312,"data_stored":443653,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40699999999999997,"15min":0.41099999999999998},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40000000000000002},"min":{"1min":0.28899999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.52400000000000002,"15min":0.50600000000000001},"min":{"1min":0.436,"5min":0.317,"15min":0.317},"max":{"1min":0.74299999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.54100000000000004,"15min":0.52700000000000002},"min":{"1min":0.42299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.446,"5min":1.446,"15min":1.446},"last":0.44},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.57299999999999995,"15min":0.59099999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70399999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.627},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.63},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.624,"15min":0.63400000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.78400000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Apr 6 14:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.65700000000000003,"15min":0.65500000000000003},"min":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.53600000000000003}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.495,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.79900000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.66100000000000003,"15min":0.65400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.497}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.63200000000000001,"15min":0.64400000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.68700000000000006,"15min":0.68100000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.1379999999999999,"5min":4.1379999999999999,"15min":4.1379999999999999},"last":0.64300000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.66900000000000004,"15min":0.65900000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.68799999999999994,"15min":0.68400000000000005},"min":{"1min":0.56100000000000005,"5min":0.503,"15min":0.503},"max":{"1min":1.6499999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.58799999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27968,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717184,"statfs":{"total":95995035648,"available":95966396416,"internally_reserved":0,"allocated":974848,"data_stored":511168,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.35899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.48799999999999999,"15min":0.47299999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.56000000000000005,"15min":0.51000000000000001},"min":{"1min":0.46000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81499999999999995,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.55500000000000005,"15min":0.51700000000000002},"min":{"1min":0.47599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77500000000000002,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52600000000000002,"15min":0.49299999999999999},"min":{"1min":0.38900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.51300000000000001},"min":{"1min":0.46800000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69399999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46100000000000002,"15min":0.44700000000000001},"min":{"1min":0.38500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.41799999999999998},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.59599999999999997,"15min":0.57699999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.871,"5min":0.877,"15min":0.877},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.59099999999999997,"15min":0.56499999999999995},"min":{"1min":0.441,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.79900000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.60299999999999998,"15min":0.58999999999999997},"min":{"1min":0.48699999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81899999999999995,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.47299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.86299999999999999,"5min":0.874,"15min":0.874},"last":0.66600000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.57199999999999995,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64000000000000001,"15min":0.62},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.875,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.64100000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.70099999999999996,"15min":0.69499999999999995},"min":{"1min":0.53400000000000003,"5min":0.503,"15min":0.503},"max":{"1min":0.90700000000000003,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.65700000000000003,"15min":0.65000000000000002},"min":{"1min":0.56499999999999995,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.83799999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.60699999999999998}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.68000000000000005,"15min":0.67600000000000005},"min":{"1min":0.5,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.86399999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.624},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.748,"15min":0.74299999999999999},"min":{"1min":0.56999999999999995,"5min":0.502,"15min":0.502},"max":{"1min":0.91400000000000003,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.70599999999999996}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.71399999999999997,"15min":0.71099999999999997},"min":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.88,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.72299999999999998,"15min":0.71699999999999997},"min":{"1min":0.56799999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.876,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.67600000000000005}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.65200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27924,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717228,"statfs":{"total":95995035648,"available":95966441472,"internally_reserved":0,"allocated":929792,"data_stored":444924,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.49299999999999999,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66400000000000003,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.498,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.624,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57099999999999995,"15min":0.56699999999999995},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.56899999999999995},"min":{"1min":0.42499999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69199999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.435,"5min":0.435,"15min":0.435},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.497,"15min":0.499},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65100000000000002,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.375},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60899999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32600000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.45400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78700000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.44900000000000001,"15min":0.46300000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.499,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.46899999999999997,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.55900000000000005,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.502,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.65500000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.58299999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.57999999999999996,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.55100000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.54700000000000004,"15min":0.56100000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.627,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.432}]},{"osd":10,"last update":"Sat Apr 6 14:05:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.68100000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64100000000000001,"15min":0.64900000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.72299999999999998,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.49399999999999999}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":976,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":999424,"data_stored":512863,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.51300000000000001,"15min":0.46600000000000003},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.46899999999999997,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72999999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79300000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56100000000000005,"15min":0.52600000000000002},"min":{"1min":0.45200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79600000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.443}]},{"osd":3,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.58499999999999996,"15min":0.55500000000000005},"min":{"1min":0.45100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.391,"5min":3.391,"15min":3.391},"last":0.5},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56499999999999995,"15min":0.54200000000000004},"min":{"1min":0.46500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.109,"5min":2.109,"15min":2.109},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.45600000000000002,"15min":0.436},"min":{"1min":0.307,"5min":0.216,"15min":0.216},"max":{"1min":0.78400000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.44400000000000001,"15min":0.42199999999999999},"min":{"1min":0.34999999999999998,"5min":0.214,"15min":0.214},"max":{"1min":0.66800000000000004,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60999999999999999,"15min":0.60099999999999998},"min":{"1min":0.39500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.82599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59499999999999997,"15min":0.57999999999999996},"min":{"1min":0.51300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.45200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.71499999999999997,"15min":0.71099999999999997},"min":{"1min":0.57299999999999995,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.90600000000000003,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.66500000000000004,"15min":0.65500000000000003},"min":{"1min":0.46700000000000003,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53400000000000003}]},{"osd":10,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.74199999999999999,"15min":0.72299999999999998},"min":{"1min":0.63100000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.74199999999999999,"15min":0.72699999999999998},"min":{"1min":0.61499999999999999,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.7799999999999998,"5min":2.7799999999999998,"15min":2.7799999999999998},"last":0.57599999999999996}]},{"osd":11,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56000000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36156,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93708996,"statfs":{"total":95995035648,"available":95958011904,"internally_reserved":0,"allocated":970752,"data_stored":968780,"data_compressed":28220,"data_compressed_allocated":36864,"data_compressed_original":524288,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.33700000000000002,"15min":0.33300000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.42899999999999999,"5min":0.629,"15min":0.629},"last":0.314},{"interface":"front","average":{"1min":0.32300000000000001,"5min":0.34200000000000003,"15min":0.33200000000000002},"min":{"1min":0.253,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.41699999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68999999999999995,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51300000000000001,"15min":0.49099999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70599999999999996,"5min":0.872,"15min":0.872},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.66200000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.436,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.63500000000000001,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.82099999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65200000000000002,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.45500000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.72699999999999998,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.64300000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.82899999999999996}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.47499999999999998,"15min":0.48499999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.55600000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.47599999999999998,"15min":0.49399999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.54300000000000004,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56100000000000005,"15min":0.56200000000000006},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.754,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.754},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76400000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.76400000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:04:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.60199999999999998,"15min":0.626},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.53800000000000003,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.50700000000000001,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.504,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.81000000000000005}]},{"osd":10,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.624,"15min":0.63600000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.72199999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.72999999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.79800000000000004}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.83799999999999997}]}]},{"osd":4,"up_from":33,"seq":141733920819,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717236,"statfs":{"total":95995035648,"available":95966449664,"internally_reserved":0,"allocated":917504,"data_stored":445472,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35399999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.40200000000000002,"15min":0.378},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.70199999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":1.036,"15min":1.036},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.55800000000000005,"15min":0.53600000000000003},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.50700000000000001},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76500000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.441},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.47999999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68500000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.57199999999999995,"15min":0.55400000000000005},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82299999999999995,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.60399999999999998,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79600000000000004,"5min":1.012,"15min":1.012},"last":0.496}]},{"osd":6,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56599999999999995,"15min":0.54200000000000004},"min":{"1min":0.45000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.72599999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54500000000000004,"15min":0.51500000000000001},"min":{"1min":0.45900000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.44900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57699999999999996,"15min":0.56299999999999994},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.35699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.68000000000000005,"15min":0.67700000000000005},"min":{"1min":0.52200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85799999999999998,"5min":0.879,"15min":0.879},"last":0.432},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73899999999999999,"15min":0.746},"min":{"1min":0.57999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81999999999999995,"5min":1.694,"15min":1.694},"last":0.45400000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.71599999999999997,"15min":0.71999999999999997},"min":{"1min":0.53300000000000003,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83299999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.62},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.69599999999999995,"15min":0.69199999999999995},"min":{"1min":0.54500000000000004,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.80400000000000005,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61099999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.76900000000000002,"15min":0.77800000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.86199999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.73199999999999998,"15min":0.73399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.48199999999999998}]},{"osd":11,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.53500000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":1003520,"data_stored":1167352,"data_compressed":44794,"data_compressed_allocated":57344,"data_compressed_original":720896,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.5,"15min":0.48299999999999998},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.70199999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.495,"15min":0.47899999999999998},"min":{"1min":0.36799999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73099999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.53500000000000003,"15min":0.505},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.52500000000000002,"15min":0.497},"min":{"1min":0.44600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.40200000000000002,"15min":0.376},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.65100000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.318},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.41799999999999998,"15min":0.38900000000000001},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":0.65700000000000003,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52500000000000002,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67600000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.54300000000000004,"15min":0.52100000000000002},"min":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.74299999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59999999999999998,"15min":0.57099999999999995},"min":{"1min":0.438,"5min":0.432,"15min":0.432},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.58099999999999996,"15min":0.57099999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59699999999999998,"15min":0.59199999999999997},"min":{"1min":0.45400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80300000000000005,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.629,"15min":0.61899999999999999},"min":{"1min":0.497,"5min":0.378,"15min":0.378},"max":{"1min":0.76400000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.435}]},{"osd":7,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.64600000000000002,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.86199999999999999,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.64100000000000001,"15min":0.623},"min":{"1min":0.48899999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.85399999999999998,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.52600000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.625,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.68400000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.88200000000000001,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.48399999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.69499999999999995,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.84399999999999997,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.753,"15min":0.75800000000000001},"min":{"1min":0.54000000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.94699999999999995,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.54900000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.60799999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.90600000000000003,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.71299999999999997,"15min":0.70699999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":67722,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":69417,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":525100,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":133421,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":393372,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394034,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":201700,"data_compressed":7989,"data_compressed_allocated":12288,"data_compressed_original":131072,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:05:29.154 INFO:tasks.fwd_scrub.ceph_manager:clean! 2024-04-06T14:05:29.154 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-04-06T14:05:29.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.311+0000 7feb77022700 1 -- 172.21.15.90:0/87846572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb70105bf0 msgr2=0x7feb70105fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.311+0000 7feb77022700 1 --2- 172.21.15.90:0/87846572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb70105bf0 0x7feb70105fd0 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 crypto rx=0x7feb640099c0 tx=0x7feb64009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 -- 172.21.15.90:0/87846572 shutdown_connections 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 --2- 172.21.15.90:0/87846572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb701004f0 0x7feb7010c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 --2- 172.21.15.90:0/87846572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb70105bf0 0x7feb70105fd0 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 --2- 172.21.15.90:0/87846572 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb700ffb30 0x7feb700fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 -- 172.21.15.90:0/87846572 >> 172.21.15.90:0/87846572 conn(0x7feb700fb5e0 msgr2=0x7feb700fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 -- 172.21.15.90:0/87846572 shutdown_connections 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 -- 172.21.15.90:0/87846572 wait complete. 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 Processor -- start 2024-04-06T14:05:29.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.312+0000 7feb77022700 1 -- start start 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb700ffb30 0x7feb70195f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb70105bf0 0x7feb7019a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb7019ae30 con 0x7feb701004f0 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb7019afa0 con 0x7feb700ffb30 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feb7019b160 con 0x7feb70105bf0 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50312/0 (socket says 172.21.15.90:50312) 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 -- 172.21.15.90:0/1110077252 learned_addr learned my addr 172.21.15.90:0/1110077252 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb755bf700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb70105bf0 0x7feb7019a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb74dbe700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb700ffb30 0x7feb70195f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 -- 172.21.15.90:0/1110077252 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb70105bf0 msgr2=0x7feb7019a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb70105bf0 0x7feb7019a7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 -- 172.21.15.90:0/1110077252 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb700ffb30 msgr2=0x7feb70195f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb700ffb30 0x7feb70195f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 -- 172.21.15.90:0/1110077252 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feb640095f0 con 0x7feb701004f0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6ffff700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7feb5c00d980 tx=0x7feb5c00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb5c004220 con 0x7feb701004f0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7feb5c007640 con 0x7feb701004f0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feb5c005690 con 0x7feb701004f0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.313+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feb7019b440 con 0x7feb701004f0 2024-04-06T14:05:29.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.314+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feb701a2c80 con 0x7feb701004f0 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.315+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feb70108070 con 0x7feb701004f0 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.315+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7feb5c004750 con 0x7feb701004f0 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.316+0000 7feb6dffb700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 0x7feb600a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.316+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7feb5c0ccd20 con 0x7feb701004f0 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.316+0000 7feb74dbe700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 0x7feb600a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.316+0000 7feb74dbe700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 0x7feb600a38f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7feb64009fd0 tx=0x7feb64019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:29.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.320+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7feb5c095f30 con 0x7feb701004f0 2024-04-06T14:05:29.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.516+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7feb70197210 con 0x7feb600a1430 2024-04-06T14:05:29.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.526+0000 7feb6dffb700 1 -- 172.21.15.90:0/1110077252 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+344496 (secure 0 0 0) 0x7feb70197210 con 0x7feb600a1430 2024-04-06T14:05:29.530 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 msgr2=0x7feb600a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 0x7feb600a38f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7feb64009fd0 tx=0x7feb64019040 comp rx=0 tx=0).stop 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 msgr2=0x7feb70196450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 crypto rx=0x7feb5c00d980 tx=0x7feb5c00dc90 comp rx=0 tx=0).stop 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 shutdown_connections 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7feb600a1430 0x7feb600a38f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feb70105bf0 0x7feb7019a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feb700ffb30 0x7feb70195f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 --2- 172.21.15.90:0/1110077252 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feb701004f0 0x7feb70196450 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 >> 172.21.15.90:0/1110077252 conn(0x7feb700fb5e0 msgr2=0x7feb700fcd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 shutdown_connections 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.529+0000 7feb77022700 1 -- 172.21.15.90:0/1110077252 wait complete. 2024-04-06T14:05:29.534 INFO:teuthology.orchestra.run.smithi090.stderr:dumped all 2024-04-06T14:05:29.542 INFO:teuthology.orchestra.run.smithi090.stdout:{"pg_ready":true,"pg_map":{"version":5,"stamp":"2024-04-06T14:05:27.571371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1187895,"num_objects":66,"num_object_clones":0,"num_object_copies":198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66,"num_whiteouts":0,"num_read":727,"num_read_kb":518,"num_write":452,"num_write_kb":4368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1181318,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":854,"ondisk_log_size":854,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":360068,"kb_used_data":11268,"kb_used_omap":18,"kb_used_meta":348653,"kb_avail":1124581756,"statfs":{"total":1151940427776,"available":1151571718144,"internally_reserved":0,"allocated":11538432,"data_stored":8884044,"data_compressed":223581,"data_compressed_allocated":294912,"data_compressed_original":3440640,"omap_allocated":19129,"internal_metadata":357020999},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.988099"},"pg_stats":[{"pgid":"3.3f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615419+0000","last_change":"2024-04-06T14:04:01.453716+0000","last_active":"2024-04-06T14:05:23.615419+0000","last_peered":"2024-04-06T14:05:23.615419+0000","last_clean":"2024-04-06T14:05:23.615419+0000","last_became_active":"2024-04-06T14:04:01.453279+0000","last_became_peered":"2024-04-06T14:04:01.453279+0000","last_unstale":"2024-04-06T14:05:23.615419+0000","last_undegraded":"2024-04-06T14:05:23.615419+0000","last_fullsized":"2024-04-06T14:05:23.615419+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:29:35.568654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607957+0000","last_change":"2024-04-06T14:04:01.466455+0000","last_active":"2024-04-06T14:05:23.607957+0000","last_peered":"2024-04-06T14:05:23.607957+0000","last_clean":"2024-04-06T14:05:23.607957+0000","last_became_active":"2024-04-06T14:04:01.466171+0000","last_became_peered":"2024-04-06T14:04:01.466171+0000","last_unstale":"2024-04-06T14:05:23.607957+0000","last_undegraded":"2024-04-06T14:05:23.607957+0000","last_fullsized":"2024-04-06T14:05:23.607957+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:29.306471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616328+0000","last_change":"2024-04-06T14:04:01.448626+0000","last_active":"2024-04-06T14:05:23.616328+0000","last_peered":"2024-04-06T14:05:23.616328+0000","last_clean":"2024-04-06T14:05:23.616328+0000","last_became_active":"2024-04-06T14:04:01.448333+0000","last_became_peered":"2024-04-06T14:04:01.448333+0000","last_unstale":"2024-04-06T14:05:23.616328+0000","last_undegraded":"2024-04-06T14:05:23.616328+0000","last_fullsized":"2024-04-06T14:05:23.616328+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:42.745782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.593301+0000","last_change":"2024-04-06T14:04:01.458837+0000","last_active":"2024-04-06T14:05:23.593301+0000","last_peered":"2024-04-06T14:05:23.593301+0000","last_clean":"2024-04-06T14:05:23.593301+0000","last_became_active":"2024-04-06T14:04:01.458329+0000","last_became_peered":"2024-04-06T14:04:01.458329+0000","last_unstale":"2024-04-06T14:05:23.593301+0000","last_undegraded":"2024-04-06T14:05:23.593301+0000","last_fullsized":"2024-04-06T14:05:23.593301+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:19:40.448569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.592862+0000","last_change":"2024-04-06T14:04:01.445957+0000","last_active":"2024-04-06T14:05:23.592862+0000","last_peered":"2024-04-06T14:05:23.592862+0000","last_clean":"2024-04-06T14:05:23.592862+0000","last_became_active":"2024-04-06T14:04:01.445603+0000","last_became_peered":"2024-04-06T14:04:01.445603+0000","last_unstale":"2024-04-06T14:05:23.592862+0000","last_undegraded":"2024-04-06T14:05:23.592862+0000","last_fullsized":"2024-04-06T14:05:23.592862+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:45.303521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604597+0000","last_change":"2024-04-06T14:04:01.453277+0000","last_active":"2024-04-06T14:05:23.604597+0000","last_peered":"2024-04-06T14:05:23.604597+0000","last_clean":"2024-04-06T14:05:23.604597+0000","last_became_active":"2024-04-06T14:04:01.452922+0000","last_became_peered":"2024-04-06T14:04:01.452922+0000","last_unstale":"2024-04-06T14:05:23.604597+0000","last_undegraded":"2024-04-06T14:05:23.604597+0000","last_fullsized":"2024-04-06T14:05:23.604597+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:45:58.260121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609927+0000","last_change":"2024-04-06T14:04:01.455808+0000","last_active":"2024-04-06T14:05:23.609927+0000","last_peered":"2024-04-06T14:05:23.609927+0000","last_clean":"2024-04-06T14:05:23.609927+0000","last_became_active":"2024-04-06T14:04:01.453251+0000","last_became_peered":"2024-04-06T14:04:01.453251+0000","last_unstale":"2024-04-06T14:05:23.609927+0000","last_undegraded":"2024-04-06T14:05:23.609927+0000","last_fullsized":"2024-04-06T14:05:23.609927+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:32.502447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603861+0000","last_change":"2024-04-06T14:04:01.452646+0000","last_active":"2024-04-06T14:05:23.603861+0000","last_peered":"2024-04-06T14:05:23.603861+0000","last_clean":"2024-04-06T14:05:23.603861+0000","last_became_active":"2024-04-06T14:04:01.452212+0000","last_became_peered":"2024-04-06T14:04:01.452212+0000","last_unstale":"2024-04-06T14:05:23.603861+0000","last_undegraded":"2024-04-06T14:05:23.603861+0000","last_fullsized":"2024-04-06T14:05:23.603861+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:26:15.098829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603421+0000","last_change":"2024-04-06T14:04:01.458389+0000","last_active":"2024-04-06T14:05:23.603421+0000","last_peered":"2024-04-06T14:05:23.603421+0000","last_clean":"2024-04-06T14:05:23.603421+0000","last_became_active":"2024-04-06T14:04:01.456662+0000","last_became_peered":"2024-04-06T14:04:01.456662+0000","last_unstale":"2024-04-06T14:05:23.603421+0000","last_undegraded":"2024-04-06T14:05:23.603421+0000","last_fullsized":"2024-04-06T14:05:23.603421+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:54.439513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"84'16","reported_seq":30,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.603913+0000","last_change":"2024-04-06T14:04:01.459377+0000","last_active":"2024-04-06T14:05:23.603913+0000","last_peered":"2024-04-06T14:05:23.603913+0000","last_clean":"2024-04-06T14:05:23.603913+0000","last_became_active":"2024-04-06T14:04:01.459077+0000","last_became_peered":"2024-04-06T14:04:01.459077+0000","last_unstale":"2024-04-06T14:05:23.603913+0000","last_undegraded":"2024-04-06T14:05:23.603913+0000","last_fullsized":"2024-04-06T14:05:23.603913+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:27:43.045616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609304+0000","last_change":"2024-04-06T14:04:01.465742+0000","last_active":"2024-04-06T14:05:23.609304+0000","last_peered":"2024-04-06T14:05:23.609304+0000","last_clean":"2024-04-06T14:05:23.609304+0000","last_became_active":"2024-04-06T14:04:01.465360+0000","last_became_peered":"2024-04-06T14:04:01.465360+0000","last_unstale":"2024-04-06T14:05:23.609304+0000","last_undegraded":"2024-04-06T14:05:23.609304+0000","last_fullsized":"2024-04-06T14:05:23.609304+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:38:41.190045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602535+0000","last_change":"2024-04-06T14:04:01.456990+0000","last_active":"2024-04-06T14:05:23.602535+0000","last_peered":"2024-04-06T14:05:23.602535+0000","last_clean":"2024-04-06T14:05:23.602535+0000","last_became_active":"2024-04-06T14:04:01.456372+0000","last_became_peered":"2024-04-06T14:04:01.456372+0000","last_unstale":"2024-04-06T14:05:23.602535+0000","last_undegraded":"2024-04-06T14:05:23.602535+0000","last_fullsized":"2024-04-06T14:05:23.602535+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:24:08.126640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610171+0000","last_change":"2024-04-06T14:04:01.448140+0000","last_active":"2024-04-06T14:05:23.610171+0000","last_peered":"2024-04-06T14:05:23.610171+0000","last_clean":"2024-04-06T14:05:23.610171+0000","last_became_active":"2024-04-06T14:04:01.447845+0000","last_became_peered":"2024-04-06T14:04:01.447845+0000","last_unstale":"2024-04-06T14:05:23.610171+0000","last_undegraded":"2024-04-06T14:05:23.610171+0000","last_fullsized":"2024-04-06T14:05:23.610171+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:00:35.068723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611310+0000","last_change":"2024-04-06T14:04:01.449217+0000","last_active":"2024-04-06T14:05:23.611310+0000","last_peered":"2024-04-06T14:05:23.611310+0000","last_clean":"2024-04-06T14:05:23.611310+0000","last_became_active":"2024-04-06T14:04:01.448620+0000","last_became_peered":"2024-04-06T14:04:01.448620+0000","last_unstale":"2024-04-06T14:05:23.611310+0000","last_undegraded":"2024-04-06T14:05:23.611310+0000","last_fullsized":"2024-04-06T14:05:23.611310+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:04:41.307400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614994+0000","last_change":"2024-04-06T14:04:01.457129+0000","last_active":"2024-04-06T14:05:23.614994+0000","last_peered":"2024-04-06T14:05:23.614994+0000","last_clean":"2024-04-06T14:05:23.614994+0000","last_became_active":"2024-04-06T14:04:01.456858+0000","last_became_peered":"2024-04-06T14:04:01.456858+0000","last_unstale":"2024-04-06T14:05:23.614994+0000","last_undegraded":"2024-04-06T14:05:23.614994+0000","last_fullsized":"2024-04-06T14:05:23.614994+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:01:10.564373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611373+0000","last_change":"2024-04-06T14:04:01.464903+0000","last_active":"2024-04-06T14:05:23.611373+0000","last_peered":"2024-04-06T14:05:23.611373+0000","last_clean":"2024-04-06T14:05:23.611373+0000","last_became_active":"2024-04-06T14:04:01.464578+0000","last_became_peered":"2024-04-06T14:04:01.464578+0000","last_unstale":"2024-04-06T14:05:23.611373+0000","last_undegraded":"2024-04-06T14:05:23.611373+0000","last_fullsized":"2024-04-06T14:05:23.611373+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:35:52.200104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615524+0000","last_change":"2024-04-06T14:04:01.452041+0000","last_active":"2024-04-06T14:05:23.615524+0000","last_peered":"2024-04-06T14:05:23.615524+0000","last_clean":"2024-04-06T14:05:23.615524+0000","last_became_active":"2024-04-06T14:04:01.451641+0000","last_became_peered":"2024-04-06T14:04:01.451641+0000","last_unstale":"2024-04-06T14:05:23.615524+0000","last_undegraded":"2024-04-06T14:05:23.615524+0000","last_fullsized":"2024-04-06T14:05:23.615524+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:18:29.150261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613532+0000","last_change":"2024-04-06T14:04:01.452092+0000","last_active":"2024-04-06T14:05:23.613532+0000","last_peered":"2024-04-06T14:05:23.613532+0000","last_clean":"2024-04-06T14:05:23.613532+0000","last_became_active":"2024-04-06T14:04:01.451621+0000","last_became_peered":"2024-04-06T14:04:01.451621+0000","last_unstale":"2024-04-06T14:05:23.613532+0000","last_undegraded":"2024-04-06T14:05:23.613532+0000","last_fullsized":"2024-04-06T14:05:23.613532+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:20:43.462307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611801+0000","last_change":"2024-04-06T14:04:01.452258+0000","last_active":"2024-04-06T14:05:23.611801+0000","last_peered":"2024-04-06T14:05:23.611801+0000","last_clean":"2024-04-06T14:05:23.611801+0000","last_became_active":"2024-04-06T14:04:01.451818+0000","last_became_peered":"2024-04-06T14:04:01.451818+0000","last_unstale":"2024-04-06T14:05:23.611801+0000","last_undegraded":"2024-04-06T14:05:23.611801+0000","last_fullsized":"2024-04-06T14:05:23.611801+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:13:42.559604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"81'9","reported_seq":44,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.235666+0000","last_change":"2024-04-06T14:05:05.235370+0000","last_active":"2024-04-06T14:05:05.235666+0000","last_peered":"2024-04-06T14:05:05.235666+0000","last_clean":"2024-04-06T14:05:05.235666+0000","last_became_active":"2024-04-06T14:05:05.229253+0000","last_became_peered":"2024-04-06T14:05:05.229253+0000","last_unstale":"2024-04-06T14:05:05.235666+0000","last_undegraded":"2024-04-06T14:05:05.235666+0000","last_fullsized":"2024-04-06T14:05:05.235666+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:37:23.805018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,4],"acting":[9,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596468+0000","last_change":"2024-04-06T14:04:01.458831+0000","last_active":"2024-04-06T14:05:23.596468+0000","last_peered":"2024-04-06T14:05:23.596468+0000","last_clean":"2024-04-06T14:05:23.596468+0000","last_became_active":"2024-04-06T14:04:01.458535+0000","last_became_peered":"2024-04-06T14:04:01.458535+0000","last_unstale":"2024-04-06T14:05:23.596468+0000","last_undegraded":"2024-04-06T14:05:23.596468+0000","last_fullsized":"2024-04-06T14:05:23.596468+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:35:18.369855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1e","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190339+0000","last_change":"2024-04-06T14:04:01.033537+0000","last_active":"2024-04-06T14:05:05.190339+0000","last_peered":"2024-04-06T14:05:05.190339+0000","last_clean":"2024-04-06T14:05:05.190339+0000","last_became_active":"2024-04-06T14:04:01.033179+0000","last_became_peered":"2024-04-06T14:04:01.033179+0000","last_unstale":"2024-04-06T14:05:05.190339+0000","last_undegraded":"2024-04-06T14:05:05.190339+0000","last_fullsized":"2024-04-06T14:05:05.190339+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:51:59.673580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617749+0000","last_change":"2024-04-06T14:04:01.445558+0000","last_active":"2024-04-06T14:05:23.617749+0000","last_peered":"2024-04-06T14:05:23.617749+0000","last_clean":"2024-04-06T14:05:23.617749+0000","last_became_active":"2024-04-06T14:04:01.445233+0000","last_became_peered":"2024-04-06T14:04:01.445233+0000","last_unstale":"2024-04-06T14:05:23.617749+0000","last_undegraded":"2024-04-06T14:05:23.617749+0000","last_fullsized":"2024-04-06T14:05:23.617749+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:48:18.320903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1d","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196231+0000","last_change":"2024-04-06T14:04:01.039284+0000","last_active":"2024-04-06T14:05:05.196231+0000","last_peered":"2024-04-06T14:05:05.196231+0000","last_clean":"2024-04-06T14:05:05.196231+0000","last_became_active":"2024-04-06T14:04:01.038911+0000","last_became_peered":"2024-04-06T14:04:01.038911+0000","last_unstale":"2024-04-06T14:05:05.196231+0000","last_undegraded":"2024-04-06T14:05:05.196231+0000","last_fullsized":"2024-04-06T14:05:05.196231+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:24.968468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607848+0000","last_change":"2024-04-06T14:04:01.455335+0000","last_active":"2024-04-06T14:05:23.607848+0000","last_peered":"2024-04-06T14:05:23.607848+0000","last_clean":"2024-04-06T14:05:23.607848+0000","last_became_active":"2024-04-06T14:04:01.455039+0000","last_became_peered":"2024-04-06T14:04:01.455039+0000","last_unstale":"2024-04-06T14:05:23.607848+0000","last_undegraded":"2024-04-06T14:05:23.607848+0000","last_fullsized":"2024-04-06T14:05:23.607848+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:24:53.317024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1c","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195822+0000","last_change":"2024-04-06T14:04:01.038355+0000","last_active":"2024-04-06T14:05:05.195822+0000","last_peered":"2024-04-06T14:05:05.195822+0000","last_clean":"2024-04-06T14:05:05.195822+0000","last_became_active":"2024-04-06T14:04:01.037982+0000","last_became_peered":"2024-04-06T14:04:01.037982+0000","last_unstale":"2024-04-06T14:05:05.195822+0000","last_undegraded":"2024-04-06T14:05:05.195822+0000","last_fullsized":"2024-04-06T14:05:05.195822+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:44:16.855105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"84'15","reported_seq":38,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600996+0000","last_change":"2024-04-06T14:05:05.196872+0000","last_active":"2024-04-06T14:05:23.600996+0000","last_peered":"2024-04-06T14:05:23.600996+0000","last_clean":"2024-04-06T14:05:23.600996+0000","last_became_active":"2024-04-06T14:05:05.196522+0000","last_became_peered":"2024-04-06T14:05:05.196522+0000","last_unstale":"2024-04-06T14:05:23.600996+0000","last_undegraded":"2024-04-06T14:05:23.600996+0000","last_fullsized":"2024-04-06T14:05:23.600996+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:12:54.386682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,1],"acting":[5,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1b","version":"84'6","reported_seq":101,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626416+0000","last_change":"2024-04-06T14:04:01.034603+0000","last_active":"2024-04-06T14:05:23.626416+0000","last_peered":"2024-04-06T14:05:23.626416+0000","last_clean":"2024-04-06T14:05:23.626416+0000","last_became_active":"2024-04-06T14:04:01.034208+0000","last_became_peered":"2024-04-06T14:04:01.034208+0000","last_unstale":"2024-04-06T14:05:23.626416+0000","last_undegraded":"2024-04-06T14:05:23.626416+0000","last_fullsized":"2024-04-06T14:05:23.626416+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:03:07.604749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":967,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":80,"num_read_kb":2,"num_write":3,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607842+0000","last_change":"2024-04-06T14:04:01.440761+0000","last_active":"2024-04-06T14:05:23.607842+0000","last_peered":"2024-04-06T14:05:23.607842+0000","last_clean":"2024-04-06T14:05:23.607842+0000","last_became_active":"2024-04-06T14:04:01.440304+0000","last_became_peered":"2024-04-06T14:04:01.440304+0000","last_unstale":"2024-04-06T14:05:23.607842+0000","last_undegraded":"2024-04-06T14:05:23.607842+0000","last_fullsized":"2024-04-06T14:05:23.607842+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:43:31.819089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.1a","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.259157+0000","last_change":"2024-04-06T14:04:01.031621+0000","last_active":"2024-04-06T14:05:05.259157+0000","last_peered":"2024-04-06T14:05:05.259157+0000","last_clean":"2024-04-06T14:05:05.259157+0000","last_became_active":"2024-04-06T14:04:01.031147+0000","last_became_peered":"2024-04-06T14:04:01.031147+0000","last_unstale":"2024-04-06T14:05:05.259157+0000","last_undegraded":"2024-04-06T14:05:05.259157+0000","last_fullsized":"2024-04-06T14:05:05.259157+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:20:55.646453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.599735+0000","last_change":"2024-04-06T14:04:01.457952+0000","last_active":"2024-04-06T14:05:23.599735+0000","last_peered":"2024-04-06T14:05:23.599735+0000","last_clean":"2024-04-06T14:05:23.599735+0000","last_became_active":"2024-04-06T14:04:01.457639+0000","last_became_peered":"2024-04-06T14:04:01.457639+0000","last_unstale":"2024-04-06T14:05:23.599735+0000","last_undegraded":"2024-04-06T14:05:23.599735+0000","last_fullsized":"2024-04-06T14:05:23.599735+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:33:24.900407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.19","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193183+0000","last_change":"2024-04-06T14:04:01.030829+0000","last_active":"2024-04-06T14:05:05.193183+0000","last_peered":"2024-04-06T14:05:05.193183+0000","last_clean":"2024-04-06T14:05:05.193183+0000","last_became_active":"2024-04-06T14:04:01.030476+0000","last_became_peered":"2024-04-06T14:04:01.030476+0000","last_unstale":"2024-04-06T14:05:05.193183+0000","last_undegraded":"2024-04-06T14:05:05.193183+0000","last_fullsized":"2024-04-06T14:05:05.193183+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:25:10.318661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604741+0000","last_change":"2024-04-06T14:04:01.451771+0000","last_active":"2024-04-06T14:05:23.604741+0000","last_peered":"2024-04-06T14:05:23.604741+0000","last_clean":"2024-04-06T14:05:23.604741+0000","last_became_active":"2024-04-06T14:04:01.451470+0000","last_became_peered":"2024-04-06T14:04:01.451470+0000","last_unstale":"2024-04-06T14:05:23.604741+0000","last_undegraded":"2024-04-06T14:05:23.604741+0000","last_fullsized":"2024-04-06T14:05:23.604741+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:36:49.011004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189600+0000","last_change":"2024-04-06T14:04:01.038698+0000","last_active":"2024-04-06T14:05:05.189600+0000","last_peered":"2024-04-06T14:05:05.189600+0000","last_clean":"2024-04-06T14:05:05.189600+0000","last_became_active":"2024-04-06T14:04:01.037017+0000","last_became_peered":"2024-04-06T14:04:01.037017+0000","last_unstale":"2024-04-06T14:05:05.189600+0000","last_undegraded":"2024-04-06T14:05:05.189600+0000","last_fullsized":"2024-04-06T14:05:05.189600+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:14:45.020853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.607641+0000","last_change":"2024-04-06T14:04:01.447348+0000","last_active":"2024-04-06T14:05:23.607641+0000","last_peered":"2024-04-06T14:05:23.607641+0000","last_clean":"2024-04-06T14:05:23.607641+0000","last_became_active":"2024-04-06T14:04:01.447030+0000","last_became_peered":"2024-04-06T14:04:01.447030+0000","last_unstale":"2024-04-06T14:05:23.607641+0000","last_undegraded":"2024-04-06T14:05:23.607641+0000","last_fullsized":"2024-04-06T14:05:23.607641+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:42:20.640119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.17","version":"84'4","reported_seq":97,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.626088+0000","last_change":"2024-04-06T14:04:01.030905+0000","last_active":"2024-04-06T14:05:23.626088+0000","last_peered":"2024-04-06T14:05:23.626088+0000","last_clean":"2024-04-06T14:05:23.626088+0000","last_became_active":"2024-04-06T14:04:01.030472+0000","last_became_peered":"2024-04-06T14:04:01.030472+0000","last_unstale":"2024-04-06T14:05:23.626088+0000","last_undegraded":"2024-04-06T14:05:23.626088+0000","last_fullsized":"2024-04-06T14:05:23.626088+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:19:38.425552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.614466+0000","last_change":"2024-04-06T14:04:01.466584+0000","last_active":"2024-04-06T14:05:23.614466+0000","last_peered":"2024-04-06T14:05:23.614466+0000","last_clean":"2024-04-06T14:05:23.614466+0000","last_became_active":"2024-04-06T14:04:01.466277+0000","last_became_peered":"2024-04-06T14:04:01.466277+0000","last_unstale":"2024-04-06T14:05:23.614466+0000","last_undegraded":"2024-04-06T14:05:23.614466+0000","last_fullsized":"2024-04-06T14:05:23.614466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:34:56.881548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.16","version":"81'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194821+0000","last_change":"2024-04-06T14:04:01.034396+0000","last_active":"2024-04-06T14:05:05.194821+0000","last_peered":"2024-04-06T14:05:05.194821+0000","last_clean":"2024-04-06T14:05:05.194821+0000","last_became_active":"2024-04-06T14:04:01.034055+0000","last_became_peered":"2024-04-06T14:04:01.034055+0000","last_unstale":"2024-04-06T14:05:05.194821+0000","last_undegraded":"2024-04-06T14:05:05.194821+0000","last_fullsized":"2024-04-06T14:05:05.194821+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:07:41.573703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610283+0000","last_change":"2024-04-06T14:04:01.453508+0000","last_active":"2024-04-06T14:05:23.610283+0000","last_peered":"2024-04-06T14:05:23.610283+0000","last_clean":"2024-04-06T14:05:23.610283+0000","last_became_active":"2024-04-06T14:04:01.452683+0000","last_became_peered":"2024-04-06T14:04:01.452683+0000","last_unstale":"2024-04-06T14:05:23.610283+0000","last_undegraded":"2024-04-06T14:05:23.610283+0000","last_fullsized":"2024-04-06T14:05:23.610283+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:58:32.088181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.190092+0000","last_change":"2024-04-06T14:04:01.040198+0000","last_active":"2024-04-06T14:05:05.190092+0000","last_peered":"2024-04-06T14:05:05.190092+0000","last_clean":"2024-04-06T14:05:05.190092+0000","last_became_active":"2024-04-06T14:04:01.039786+0000","last_became_peered":"2024-04-06T14:04:01.039786+0000","last_unstale":"2024-04-06T14:05:05.190092+0000","last_undegraded":"2024-04-06T14:05:05.190092+0000","last_fullsized":"2024-04-06T14:05:05.190092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:38:42.617640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604056+0000","last_change":"2024-04-06T14:04:01.458133+0000","last_active":"2024-04-06T14:05:23.604056+0000","last_peered":"2024-04-06T14:05:23.604056+0000","last_clean":"2024-04-06T14:05:23.604056+0000","last_became_active":"2024-04-06T14:04:01.457786+0000","last_became_peered":"2024-04-06T14:04:01.457786+0000","last_unstale":"2024-04-06T14:05:23.604056+0000","last_undegraded":"2024-04-06T14:05:23.604056+0000","last_fullsized":"2024-04-06T14:05:23.604056+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:17.152298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,4],"acting":[1,8,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.14","version":"84'34","reported_seq":215,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.840919+0000","last_change":"2024-04-06T14:04:01.034904+0000","last_active":"2024-04-06T14:05:23.840919+0000","last_peered":"2024-04-06T14:05:23.840919+0000","last_clean":"2024-04-06T14:05:23.840919+0000","last_became_active":"2024-04-06T14:04:01.034596+0000","last_became_peered":"2024-04-06T14:04:01.034596+0000","last_unstale":"2024-04-06T14:05:23.840919+0000","last_undegraded":"2024-04-06T14:05:23.840919+0000","last_fullsized":"2024-04-06T14:05:23.840919+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:11:21.405174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67722,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":166,"num_read_kb":143,"num_write":35,"num_write_kb":80,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605924+0000","last_change":"2024-04-06T14:04:01.463094+0000","last_active":"2024-04-06T14:05:23.605924+0000","last_peered":"2024-04-06T14:05:23.605924+0000","last_clean":"2024-04-06T14:05:23.605924+0000","last_became_active":"2024-04-06T14:04:01.462824+0000","last_became_peered":"2024-04-06T14:04:01.462824+0000","last_unstale":"2024-04-06T14:05:23.605924+0000","last_undegraded":"2024-04-06T14:05:23.605924+0000","last_fullsized":"2024-04-06T14:05:23.605924+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:00:35.803296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.13","version":"81'6","reported_seq":22,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191794+0000","last_change":"2024-04-06T14:04:01.034351+0000","last_active":"2024-04-06T14:05:05.191794+0000","last_peered":"2024-04-06T14:05:05.191794+0000","last_clean":"2024-04-06T14:05:05.191794+0000","last_became_active":"2024-04-06T14:04:01.033976+0000","last_became_peered":"2024-04-06T14:04:01.033976+0000","last_unstale":"2024-04-06T14:05:05.191794+0000","last_undegraded":"2024-04-06T14:05:05.191794+0000","last_fullsized":"2024-04-06T14:05:05.191794+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:47:02.518712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612189+0000","last_change":"2024-04-06T14:04:01.462285+0000","last_active":"2024-04-06T14:05:23.612189+0000","last_peered":"2024-04-06T14:05:23.612189+0000","last_clean":"2024-04-06T14:05:23.612189+0000","last_became_active":"2024-04-06T14:04:01.461936+0000","last_became_peered":"2024-04-06T14:04:01.461936+0000","last_unstale":"2024-04-06T14:05:23.612189+0000","last_undegraded":"2024-04-06T14:05:23.612189+0000","last_fullsized":"2024-04-06T14:05:23.612189+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:54:49.977696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.12","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189874+0000","last_change":"2024-04-06T14:04:01.036508+0000","last_active":"2024-04-06T14:05:05.189874+0000","last_peered":"2024-04-06T14:05:05.189874+0000","last_clean":"2024-04-06T14:05:05.189874+0000","last_became_active":"2024-04-06T14:04:01.036162+0000","last_became_peered":"2024-04-06T14:04:01.036162+0000","last_unstale":"2024-04-06T14:05:05.189874+0000","last_undegraded":"2024-04-06T14:05:05.189874+0000","last_fullsized":"2024-04-06T14:05:05.189874+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:44:02.033949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610716+0000","last_change":"2024-04-06T14:04:01.459583+0000","last_active":"2024-04-06T14:05:23.610716+0000","last_peered":"2024-04-06T14:05:23.610716+0000","last_clean":"2024-04-06T14:05:23.610716+0000","last_became_active":"2024-04-06T14:04:01.459269+0000","last_became_peered":"2024-04-06T14:04:01.459269+0000","last_unstale":"2024-04-06T14:05:23.610716+0000","last_undegraded":"2024-04-06T14:05:23.610716+0000","last_fullsized":"2024-04-06T14:05:23.610716+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:56:12.137872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"84'6","reported_seq":104,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.629802+0000","last_change":"2024-04-06T14:04:01.037008+0000","last_active":"2024-04-06T14:05:23.629802+0000","last_peered":"2024-04-06T14:05:23.629802+0000","last_clean":"2024-04-06T14:05:23.629802+0000","last_became_active":"2024-04-06T14:04:01.036662+0000","last_became_peered":"2024-04-06T14:04:01.036662+0000","last_unstale":"2024-04-06T14:05:23.629802+0000","last_undegraded":"2024-04-06T14:05:23.629802+0000","last_fullsized":"2024-04-06T14:05:23.629802+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:48:11.916044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1388,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":84,"num_read_kb":6,"num_write":5,"num_write_kb":4,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.605694+0000","last_change":"2024-04-06T14:04:01.462967+0000","last_active":"2024-04-06T14:05:23.605694+0000","last_peered":"2024-04-06T14:05:23.605694+0000","last_clean":"2024-04-06T14:05:23.605694+0000","last_became_active":"2024-04-06T14:04:01.462603+0000","last_became_peered":"2024-04-06T14:04:01.462603+0000","last_unstale":"2024-04-06T14:05:23.605694+0000","last_undegraded":"2024-04-06T14:05:23.605694+0000","last_fullsized":"2024-04-06T14:05:23.605694+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:21:04.577310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.10","version":"81'15","reported_seq":31,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195316+0000","last_change":"2024-04-06T14:04:01.038886+0000","last_active":"2024-04-06T14:05:05.195316+0000","last_peered":"2024-04-06T14:05:05.195316+0000","last_clean":"2024-04-06T14:05:05.195316+0000","last_became_active":"2024-04-06T14:04:01.038174+0000","last_became_peered":"2024-04-06T14:04:01.038174+0000","last_unstale":"2024-04-06T14:05:05.195316+0000","last_undegraded":"2024-04-06T14:05:05.195316+0000","last_fullsized":"2024-04-06T14:05:05.195316+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:03:27.909559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.602036+0000","last_change":"2024-04-06T14:04:01.458063+0000","last_active":"2024-04-06T14:05:23.602036+0000","last_peered":"2024-04-06T14:05:23.602036+0000","last_clean":"2024-04-06T14:05:23.602036+0000","last_became_active":"2024-04-06T14:04:01.457700+0000","last_became_peered":"2024-04-06T14:04:01.457700+0000","last_unstale":"2024-04-06T14:05:23.602036+0000","last_undegraded":"2024-04-06T14:05:23.602036+0000","last_fullsized":"2024-04-06T14:05:23.602036+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:46:46.494918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"81'2","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196332+0000","last_change":"2024-04-06T14:04:01.034126+0000","last_active":"2024-04-06T14:05:05.196332+0000","last_peered":"2024-04-06T14:05:05.196332+0000","last_clean":"2024-04-06T14:05:05.196332+0000","last_became_active":"2024-04-06T14:04:01.033822+0000","last_became_peered":"2024-04-06T14:04:01.033822+0000","last_unstale":"2024-04-06T14:05:05.196332+0000","last_undegraded":"2024-04-06T14:05:05.196332+0000","last_fullsized":"2024-04-06T14:05:05.196332+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:40:38.295186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604329+0000","last_change":"2024-04-06T14:05:05.196327+0000","last_active":"2024-04-06T14:05:23.604329+0000","last_peered":"2024-04-06T14:05:23.604329+0000","last_clean":"2024-04-06T14:05:23.604329+0000","last_became_active":"2024-04-06T14:05:05.195845+0000","last_became_peered":"2024-04-06T14:05:05.195845+0000","last_unstale":"2024-04-06T14:05:23.604329+0000","last_undegraded":"2024-04-06T14:05:23.604329+0000","last_fullsized":"2024-04-06T14:05:23.604329+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:16:06.262603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,6],"acting":[11,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191665+0000","last_change":"2024-04-06T14:04:01.034767+0000","last_active":"2024-04-06T14:05:05.191665+0000","last_peered":"2024-04-06T14:05:05.191665+0000","last_clean":"2024-04-06T14:05:05.191665+0000","last_became_active":"2024-04-06T14:04:01.034458+0000","last_became_peered":"2024-04-06T14:04:01.034458+0000","last_unstale":"2024-04-06T14:05:05.191665+0000","last_undegraded":"2024-04-06T14:05:05.191665+0000","last_fullsized":"2024-04-06T14:05:05.191665+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:39:02.110520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.597349+0000","last_change":"2024-04-06T14:04:01.455467+0000","last_active":"2024-04-06T14:05:23.597349+0000","last_peered":"2024-04-06T14:05:23.597349+0000","last_clean":"2024-04-06T14:05:23.597349+0000","last_became_active":"2024-04-06T14:04:01.455169+0000","last_became_peered":"2024-04-06T14:04:01.455169+0000","last_unstale":"2024-04-06T14:05:23.597349+0000","last_undegraded":"2024-04-06T14:05:23.597349+0000","last_fullsized":"2024-04-06T14:05:23.597349+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:35:10.727032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.2","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194517+0000","last_change":"2024-04-06T14:04:01.032033+0000","last_active":"2024-04-06T14:05:05.194517+0000","last_peered":"2024-04-06T14:05:05.194517+0000","last_clean":"2024-04-06T14:05:05.194517+0000","last_became_active":"2024-04-06T14:04:01.031683+0000","last_became_peered":"2024-04-06T14:04:01.031683+0000","last_unstale":"2024-04-06T14:05:05.194517+0000","last_undegraded":"2024-04-06T14:05:05.194517+0000","last_fullsized":"2024-04-06T14:05:05.194517+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:14:19.903542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613134+0000","last_change":"2024-04-06T14:04:01.454281+0000","last_active":"2024-04-06T14:05:23.613134+0000","last_peered":"2024-04-06T14:05:23.613134+0000","last_clean":"2024-04-06T14:05:23.613134+0000","last_became_active":"2024-04-06T14:04:01.453950+0000","last_became_peered":"2024-04-06T14:04:01.453950+0000","last_unstale":"2024-04-06T14:05:23.613134+0000","last_undegraded":"2024-04-06T14:05:23.613134+0000","last_fullsized":"2024-04-06T14:05:23.613134+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:16:29.084635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.0","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604058+0000","last_change":"2024-04-06T14:04:01.458637+0000","last_active":"2024-04-06T14:05:23.604058+0000","last_peered":"2024-04-06T14:05:23.604058+0000","last_clean":"2024-04-06T14:05:23.604058+0000","last_became_active":"2024-04-06T14:04:01.458233+0000","last_became_peered":"2024-04-06T14:04:01.458233+0000","last_unstale":"2024-04-06T14:05:23.604058+0000","last_undegraded":"2024-04-06T14:05:23.604058+0000","last_fullsized":"2024-04-06T14:05:23.604058+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:38:57.886660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"83'5","reported_seq":98,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.576804+0000","last_change":"2024-04-06T14:04:01.031024+0000","last_active":"2024-04-06T14:05:23.576804+0000","last_peered":"2024-04-06T14:05:23.576804+0000","last_clean":"2024-04-06T14:05:23.576804+0000","last_became_active":"2024-04-06T14:04:01.030725+0000","last_became_peered":"2024-04-06T14:04:01.030725+0000","last_unstale":"2024-04-06T14:05:23.576804+0000","last_undegraded":"2024-04-06T14:05:23.576804+0000","last_fullsized":"2024-04-06T14:05:23.576804+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:12:50.240344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":78,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604005+0000","last_change":"2024-04-06T14:04:01.450845+0000","last_active":"2024-04-06T14:05:23.604005+0000","last_peered":"2024-04-06T14:05:23.604005+0000","last_clean":"2024-04-06T14:05:23.604005+0000","last_became_active":"2024-04-06T14:04:01.450365+0000","last_became_peered":"2024-04-06T14:04:01.450365+0000","last_unstale":"2024-04-06T14:05:23.604005+0000","last_undegraded":"2024-04-06T14:05:23.604005+0000","last_fullsized":"2024-04-06T14:05:23.604005+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:39:37.694883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"81'3","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228303+0000","last_change":"2024-04-06T14:04:01.033464+0000","last_active":"2024-04-06T14:05:05.228303+0000","last_peered":"2024-04-06T14:05:05.228303+0000","last_clean":"2024-04-06T14:05:05.228303+0000","last_became_active":"2024-04-06T14:04:01.033097+0000","last_became_peered":"2024-04-06T14:04:01.033097+0000","last_unstale":"2024-04-06T14:05:05.228303+0000","last_undegraded":"2024-04-06T14:05:05.228303+0000","last_fullsized":"2024-04-06T14:05:05.228303+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:58:26.258758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":4,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":337,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194265+0000","last_change":"2024-04-06T14:03:11.854078+0000","last_active":"2024-04-06T14:05:05.194265+0000","last_peered":"2024-04-06T14:05:05.194265+0000","last_clean":"2024-04-06T14:05:05.194265+0000","last_became_active":"2024-04-06T14:03:11.853639+0000","last_became_peered":"2024-04-06T14:03:11.853639+0000","last_unstale":"2024-04-06T14:05:05.194265+0000","last_undegraded":"2024-04-06T14:05:05.194265+0000","last_fullsized":"2024-04-06T14:05:05.194265+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:00:52.416242+0000","last_clean_scrub_stamp":"2024-04-06T14:00:52.416242+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:50:39.733863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"84'5","reported_seq":28,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.610576+0000","last_change":"2024-04-06T14:05:05.195151+0000","last_active":"2024-04-06T14:05:23.610576+0000","last_peered":"2024-04-06T14:05:23.610576+0000","last_clean":"2024-04-06T14:05:23.610576+0000","last_became_active":"2024-04-06T14:05:05.194828+0000","last_became_peered":"2024-04-06T14:05:05.194828+0000","last_unstale":"2024-04-06T14:05:23.610576+0000","last_undegraded":"2024-04-06T14:05:23.610576+0000","last_fullsized":"2024-04-06T14:05:23.610576+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:11:32.067936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,2],"acting":[6,9,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.194363+0000","last_change":"2024-04-06T14:04:01.029560+0000","last_active":"2024-04-06T14:05:05.194363+0000","last_peered":"2024-04-06T14:05:05.194363+0000","last_clean":"2024-04-06T14:05:05.194363+0000","last_became_active":"2024-04-06T14:04:01.027516+0000","last_became_peered":"2024-04-06T14:04:01.027516+0000","last_unstale":"2024-04-06T14:05:05.194363+0000","last_undegraded":"2024-04-06T14:05:05.194363+0000","last_fullsized":"2024-04-06T14:05:05.194363+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T23:54:52.310366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,8],"acting":[7,5,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"81'1","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.196094+0000","last_change":"2024-04-06T14:04:01.038524+0000","last_active":"2024-04-06T14:05:05.196094+0000","last_peered":"2024-04-06T14:05:05.196094+0000","last_clean":"2024-04-06T14:05:05.196094+0000","last_became_active":"2024-04-06T14:04:01.037431+0000","last_became_peered":"2024-04-06T14:04:01.037431+0000","last_unstale":"2024-04-06T14:05:05.196094+0000","last_undegraded":"2024-04-06T14:05:05.196094+0000","last_fullsized":"2024-04-06T14:05:05.196094+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:32:48.282495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609695+0000","last_change":"2024-04-06T14:04:01.450259+0000","last_active":"2024-04-06T14:05:23.609695+0000","last_peered":"2024-04-06T14:05:23.609695+0000","last_clean":"2024-04-06T14:05:23.609695+0000","last_became_active":"2024-04-06T14:04:01.450000+0000","last_became_peered":"2024-04-06T14:04:01.450000+0000","last_unstale":"2024-04-06T14:05:23.609695+0000","last_undegraded":"2024-04-06T14:05:23.609695+0000","last_fullsized":"2024-04-06T14:05:23.609695+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:46:55.870583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"84'39","reported_seq":77,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.797513+0000","last_change":"2024-04-06T14:04:01.024178+0000","last_active":"2024-04-06T14:05:23.797513+0000","last_peered":"2024-04-06T14:05:23.797513+0000","last_clean":"2024-04-06T14:05:23.797513+0000","last_became_active":"2024-04-06T14:04:01.023858+0000","last_became_peered":"2024-04-06T14:04:01.023858+0000","last_unstale":"2024-04-06T14:05:23.797513+0000","last_undegraded":"2024-04-06T14:05:23.797513+0000","last_fullsized":"2024-04-06T14:05:23.797513+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:16:19.615186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":82,"num_read_kb":70,"num_write":68,"num_write_kb":833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,3],"acting":[9,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"84'5","reported_seq":17,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.616610+0000","last_change":"2024-04-06T14:05:05.196110+0000","last_active":"2024-04-06T14:05:23.616610+0000","last_peered":"2024-04-06T14:05:23.616610+0000","last_clean":"2024-04-06T14:05:23.616610+0000","last_became_active":"2024-04-06T14:05:05.195775+0000","last_became_peered":"2024-04-06T14:05:05.195775+0000","last_unstale":"2024-04-06T14:05:23.616610+0000","last_undegraded":"2024-04-06T14:05:23.616610+0000","last_fullsized":"2024-04-06T14:05:23.616610+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:12:33.956229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,7],"acting":[9,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.7","version":"81'4","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.228555+0000","last_change":"2024-04-06T14:04:01.034440+0000","last_active":"2024-04-06T14:05:05.228555+0000","last_peered":"2024-04-06T14:05:05.228555+0000","last_clean":"2024-04-06T14:05:05.228555+0000","last_became_active":"2024-04-06T14:04:01.034122+0000","last_became_peered":"2024-04-06T14:04:01.034122+0000","last_unstale":"2024-04-06T14:05:05.228555+0000","last_undegraded":"2024-04-06T14:05:05.228555+0000","last_fullsized":"2024-04-06T14:05:05.228555+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T00:52:59.816476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.617648+0000","last_change":"2024-04-06T14:04:01.453377+0000","last_active":"2024-04-06T14:05:23.617648+0000","last_peered":"2024-04-06T14:05:23.617648+0000","last_clean":"2024-04-06T14:05:23.617648+0000","last_became_active":"2024-04-06T14:04:01.452795+0000","last_became_peered":"2024-04-06T14:04:01.452795+0000","last_unstale":"2024-04-06T14:05:23.617648+0000","last_undegraded":"2024-04-06T14:05:23.617648+0000","last_fullsized":"2024-04-06T14:05:23.617648+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:33:01.110648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"81'2","reported_seq":30,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.200728+0000","last_change":"2024-04-06T14:05:05.200365+0000","last_active":"2024-04-06T14:05:05.200728+0000","last_peered":"2024-04-06T14:05:05.200728+0000","last_clean":"2024-04-06T14:05:05.200728+0000","last_became_active":"2024-04-06T14:05:05.196172+0000","last_became_peered":"2024-04-06T14:05:05.196172+0000","last_unstale":"2024-04-06T14:05:05.200728+0000","last_undegraded":"2024-04-06T14:05:05.200728+0000","last_fullsized":"2024-04-06T14:05:05.200728+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:36:44.928650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.575363+0000","last_change":"2024-04-06T14:04:01.451264+0000","last_active":"2024-04-06T14:05:23.575363+0000","last_peered":"2024-04-06T14:05:23.575363+0000","last_clean":"2024-04-06T14:05:23.575363+0000","last_became_active":"2024-04-06T14:04:01.450869+0000","last_became_peered":"2024-04-06T14:04:01.450869+0000","last_unstale":"2024-04-06T14:05:23.575363+0000","last_undegraded":"2024-04-06T14:05:23.575363+0000","last_fullsized":"2024-04-06T14:05:23.575363+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:01:16.660448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.9","version":"81'3","reported_seq":18,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.189911+0000","last_change":"2024-04-06T14:04:01.034336+0000","last_active":"2024-04-06T14:05:05.189911+0000","last_peered":"2024-04-06T14:05:05.189911+0000","last_clean":"2024-04-06T14:05:05.189911+0000","last_became_active":"2024-04-06T14:04:01.033965+0000","last_became_peered":"2024-04-06T14:04:01.033965+0000","last_unstale":"2024-04-06T14:05:05.189911+0000","last_undegraded":"2024-04-06T14:05:05.189911+0000","last_fullsized":"2024-04-06T14:05:05.189911+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:42:26.288786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615526+0000","last_change":"2024-04-06T14:04:01.452824+0000","last_active":"2024-04-06T14:05:23.615526+0000","last_peered":"2024-04-06T14:05:23.615526+0000","last_clean":"2024-04-06T14:05:23.615526+0000","last_became_active":"2024-04-06T14:04:01.452332+0000","last_became_peered":"2024-04-06T14:04:01.452332+0000","last_unstale":"2024-04-06T14:05:23.615526+0000","last_undegraded":"2024-04-06T14:05:23.615526+0000","last_fullsized":"2024-04-06T14:05:23.615526+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:30:34.826549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.a","version":"81'1","reported_seq":16,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:06.497211+0000","last_change":"2024-04-06T14:05:05.200944+0000","last_active":"2024-04-06T14:05:06.497211+0000","last_peered":"2024-04-06T14:05:06.497211+0000","last_clean":"2024-04-06T14:05:06.497211+0000","last_became_active":"2024-04-06T14:05:05.196972+0000","last_became_peered":"2024-04-06T14:05:05.196972+0000","last_unstale":"2024-04-06T14:05:06.497211+0000","last_undegraded":"2024-04-06T14:05:06.497211+0000","last_fullsized":"2024-04-06T14:05:06.497211+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:00:15.481685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,3,8],"acting":[11,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.b","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600338+0000","last_change":"2024-04-06T14:04:01.451076+0000","last_active":"2024-04-06T14:05:23.600338+0000","last_peered":"2024-04-06T14:05:23.600338+0000","last_clean":"2024-04-06T14:05:23.600338+0000","last_became_active":"2024-04-06T14:04:01.450683+0000","last_became_peered":"2024-04-06T14:04:01.450683+0000","last_unstale":"2024-04-06T14:05:23.600338+0000","last_undegraded":"2024-04-06T14:05:23.600338+0000","last_fullsized":"2024-04-06T14:05:23.600338+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:54:56.653310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"81'4","reported_seq":20,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.195568+0000","last_change":"2024-04-06T14:04:01.039464+0000","last_active":"2024-04-06T14:05:05.195568+0000","last_peered":"2024-04-06T14:05:05.195568+0000","last_clean":"2024-04-06T14:05:05.195568+0000","last_became_active":"2024-04-06T14:04:01.038477+0000","last_became_peered":"2024-04-06T14:04:01.038477+0000","last_unstale":"2024-04-06T14:05:05.195568+0000","last_undegraded":"2024-04-06T14:05:05.195568+0000","last_fullsized":"2024-04-06T14:05:05.195568+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:17:02.269085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"84'5","reported_seq":19,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.612506+0000","last_change":"2024-04-06T14:04:01.467518+0000","last_active":"2024-04-06T14:05:23.612506+0000","last_peered":"2024-04-06T14:05:23.612506+0000","last_clean":"2024-04-06T14:05:23.612506+0000","last_became_active":"2024-04-06T14:04:01.466266+0000","last_became_peered":"2024-04-06T14:04:01.466266+0000","last_unstale":"2024-04-06T14:05:23.612506+0000","last_undegraded":"2024-04-06T14:05:23.612506+0000","last_fullsized":"2024-04-06T14:05:23.612506+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:08:33.056443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.c","version":"81'3","reported_seq":19,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.191823+0000","last_change":"2024-04-06T14:04:01.041147+0000","last_active":"2024-04-06T14:05:05.191823+0000","last_peered":"2024-04-06T14:05:05.191823+0000","last_clean":"2024-04-06T14:05:05.191823+0000","last_became_active":"2024-04-06T14:04:01.040790+0000","last_became_peered":"2024-04-06T14:04:01.040790+0000","last_unstale":"2024-04-06T14:05:05.191823+0000","last_undegraded":"2024-04-06T14:05:05.191823+0000","last_fullsized":"2024-04-06T14:05:05.191823+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:09:35.969138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.611707+0000","last_change":"2024-04-06T14:04:01.465672+0000","last_active":"2024-04-06T14:05:23.611707+0000","last_peered":"2024-04-06T14:05:23.611707+0000","last_clean":"2024-04-06T14:05:23.611707+0000","last_became_active":"2024-04-06T14:04:01.465382+0000","last_became_peered":"2024-04-06T14:04:01.465382+0000","last_unstale":"2024-04-06T14:05:23.611707+0000","last_undegraded":"2024-04-06T14:05:23.611707+0000","last_fullsized":"2024-04-06T14:05:23.611707+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:54:45.056766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"84'39","reported_seq":62,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.794092+0000","last_change":"2024-04-06T14:04:01.033442+0000","last_active":"2024-04-06T14:05:23.794092+0000","last_peered":"2024-04-06T14:05:23.794092+0000","last_clean":"2024-04-06T14:05:23.794092+0000","last_became_active":"2024-04-06T14:04:01.033075+0000","last_became_peered":"2024-04-06T14:04:01.033075+0000","last_unstale":"2024-04-06T14:05:23.794092+0000","last_undegraded":"2024-04-06T14:05:23.794092+0000","last_fullsized":"2024-04-06T14:05:23.794092+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":39,"log_dups_size":0,"ondisk_log_size":39,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:09:29.941742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131670,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":11,"num_read_kb":10,"num_write":49,"num_write_kb":475,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606466+0000","last_change":"2024-04-06T14:04:01.462267+0000","last_active":"2024-04-06T14:05:23.606466+0000","last_peered":"2024-04-06T14:05:23.606466+0000","last_clean":"2024-04-06T14:05:23.606466+0000","last_became_active":"2024-04-06T14:04:01.461950+0000","last_became_peered":"2024-04-06T14:04:01.461950+0000","last_unstale":"2024-04-06T14:05:23.606466+0000","last_undegraded":"2024-04-06T14:05:23.606466+0000","last_fullsized":"2024-04-06T14:05:23.606466+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:56:04.629063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"80'2","reported_seq":17,"reported_epoch":83,"state":"active+clean","last_fresh":"2024-04-06T14:05:05.193472+0000","last_change":"2024-04-06T14:04:01.036180+0000","last_active":"2024-04-06T14:05:05.193472+0000","last_peered":"2024-04-06T14:05:05.193472+0000","last_clean":"2024-04-06T14:05:05.193472+0000","last_became_active":"2024-04-06T14:04:01.035944+0000","last_became_peered":"2024-04-06T14:04:01.035944+0000","last_unstale":"2024-04-06T14:05:05.193472+0000","last_undegraded":"2024-04-06T14:05:05.193472+0000","last_fullsized":"2024-04-06T14:05:05.193472+0000","mapping_epoch":78,"log_start":"0'0","ondisk_log_start":"0'0","created":78,"last_epoch_clean":79,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:03:59.999063+0000","last_clean_scrub_stamp":"2024-04-06T14:03:59.999063+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:52:44.151500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"84'9","reported_seq":23,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.608430+0000","last_change":"2024-04-06T14:04:01.462682+0000","last_active":"2024-04-06T14:05:23.608430+0000","last_peered":"2024-04-06T14:05:23.608430+0000","last_clean":"2024-04-06T14:05:23.608430+0000","last_became_active":"2024-04-06T14:04:01.462319+0000","last_became_peered":"2024-04-06T14:04:01.462319+0000","last_unstale":"2024-04-06T14:05:23.608430+0000","last_undegraded":"2024-04-06T14:05:23.608430+0000","last_fullsized":"2024-04-06T14:05:23.608430+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-08T01:41:51.817593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"84'9","reported_seq":32,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.609331+0000","last_change":"2024-04-06T14:05:05.198436+0000","last_active":"2024-04-06T14:05:23.609331+0000","last_peered":"2024-04-06T14:05:23.609331+0000","last_clean":"2024-04-06T14:05:23.609331+0000","last_became_active":"2024-04-06T14:05:05.198028+0000","last_became_peered":"2024-04-06T14:05:05.198028+0000","last_unstale":"2024-04-06T14:05:23.609331+0000","last_undegraded":"2024-04-06T14:05:23.609331+0000","last_fullsized":"2024-04-06T14:05:23.609331+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:49:16.741498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,9],"acting":[11,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"84'12","reported_seq":26,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604089+0000","last_change":"2024-04-06T14:04:01.458574+0000","last_active":"2024-04-06T14:05:23.604089+0000","last_peered":"2024-04-06T14:05:23.604089+0000","last_clean":"2024-04-06T14:05:23.604089+0000","last_became_active":"2024-04-06T14:04:01.458221+0000","last_became_peered":"2024-04-06T14:04:01.458221+0000","last_unstale":"2024-04-06T14:05:23.604089+0000","last_undegraded":"2024-04-06T14:05:23.604089+0000","last_fullsized":"2024-04-06T14:05:23.604089+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:29:24.737531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615361+0000","last_change":"2024-04-06T14:04:01.453631+0000","last_active":"2024-04-06T14:05:23.615361+0000","last_peered":"2024-04-06T14:05:23.615361+0000","last_clean":"2024-04-06T14:05:23.615361+0000","last_became_active":"2024-04-06T14:04:01.452693+0000","last_became_peered":"2024-04-06T14:04:01.452693+0000","last_unstale":"2024-04-06T14:05:23.615361+0000","last_undegraded":"2024-04-06T14:05:23.615361+0000","last_fullsized":"2024-04-06T14:05:23.615361+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T18:49:03.287293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"84'7","reported_seq":21,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615434+0000","last_change":"2024-04-06T14:04:01.453149+0000","last_active":"2024-04-06T14:05:23.615434+0000","last_peered":"2024-04-06T14:05:23.615434+0000","last_clean":"2024-04-06T14:05:23.615434+0000","last_became_active":"2024-04-06T14:04:01.452831+0000","last_became_peered":"2024-04-06T14:04:01.452831+0000","last_unstale":"2024-04-06T14:05:23.615434+0000","last_undegraded":"2024-04-06T14:05:23.615434+0000","last_fullsized":"2024-04-06T14:05:23.615434+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T19:58:39.862388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"84'4","reported_seq":18,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615556+0000","last_change":"2024-04-06T14:04:01.452875+0000","last_active":"2024-04-06T14:05:23.615556+0000","last_peered":"2024-04-06T14:05:23.615556+0000","last_clean":"2024-04-06T14:05:23.615556+0000","last_became_active":"2024-04-06T14:04:01.452553+0000","last_became_peered":"2024-04-06T14:04:01.452553+0000","last_unstale":"2024-04-06T14:05:23.615556+0000","last_undegraded":"2024-04-06T14:05:23.615556+0000","last_fullsized":"2024-04-06T14:05:23.615556+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:58:10.958630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.604845+0000","last_change":"2024-04-06T14:04:01.455736+0000","last_active":"2024-04-06T14:05:23.604845+0000","last_peered":"2024-04-06T14:05:23.604845+0000","last_clean":"2024-04-06T14:05:23.604845+0000","last_became_active":"2024-04-06T14:04:01.452284+0000","last_became_peered":"2024-04-06T14:04:01.452284+0000","last_unstale":"2024-04-06T14:05:23.604845+0000","last_undegraded":"2024-04-06T14:05:23.604845+0000","last_fullsized":"2024-04-06T14:05:23.604845+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:46:18.737919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,5],"acting":[11,8,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"84'15","reported_seq":29,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.600672+0000","last_change":"2024-04-06T14:04:01.453221+0000","last_active":"2024-04-06T14:05:23.600672+0000","last_peered":"2024-04-06T14:05:23.600672+0000","last_clean":"2024-04-06T14:05:23.600672+0000","last_became_active":"2024-04-06T14:04:01.452895+0000","last_became_peered":"2024-04-06T14:04:01.452895+0000","last_unstale":"2024-04-06T14:05:23.600672+0000","last_undegraded":"2024-04-06T14:05:23.600672+0000","last_fullsized":"2024-04-06T14:05:23.600672+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T14:50:00.372345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"84'2","reported_seq":16,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.596144+0000","last_change":"2024-04-06T14:04:01.458834+0000","last_active":"2024-04-06T14:05:23.596144+0000","last_peered":"2024-04-06T14:05:23.596144+0000","last_clean":"2024-04-06T14:05:23.596144+0000","last_became_active":"2024-04-06T14:04:01.458520+0000","last_became_peered":"2024-04-06T14:04:01.458520+0000","last_unstale":"2024-04-06T14:05:23.596144+0000","last_undegraded":"2024-04-06T14:05:23.596144+0000","last_fullsized":"2024-04-06T14:05:23.596144+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T20:26:47.262148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"84'6","reported_seq":20,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606756+0000","last_change":"2024-04-06T14:04:01.451521+0000","last_active":"2024-04-06T14:05:23.606756+0000","last_peered":"2024-04-06T14:05:23.606756+0000","last_clean":"2024-04-06T14:05:23.606756+0000","last_became_active":"2024-04-06T14:04:01.451159+0000","last_became_peered":"2024-04-06T14:04:01.451159+0000","last_unstale":"2024-04-06T14:05:23.606756+0000","last_undegraded":"2024-04-06T14:05:23.606756+0000","last_fullsized":"2024-04-06T14:05:23.606756+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T16:58:03.208508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.606960+0000","last_change":"2024-04-06T14:04:01.457953+0000","last_active":"2024-04-06T14:05:23.606960+0000","last_peered":"2024-04-06T14:05:23.606960+0000","last_clean":"2024-04-06T14:05:23.606960+0000","last_became_active":"2024-04-06T14:04:01.457669+0000","last_became_peered":"2024-04-06T14:04:01.457669+0000","last_unstale":"2024-04-06T14:05:23.606960+0000","last_undegraded":"2024-04-06T14:05:23.606960+0000","last_fullsized":"2024-04-06T14:05:23.606960+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T15:22:45.851224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"84'10","reported_seq":24,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.613612+0000","last_change":"2024-04-06T14:04:01.462587+0000","last_active":"2024-04-06T14:05:23.613612+0000","last_peered":"2024-04-06T14:05:23.613612+0000","last_clean":"2024-04-06T14:05:23.613612+0000","last_became_active":"2024-04-06T14:04:01.462292+0000","last_became_peered":"2024-04-06T14:04:01.462292+0000","last_unstale":"2024-04-06T14:05:23.613612+0000","last_undegraded":"2024-04-06T14:05:23.613612+0000","last_fullsized":"2024-04-06T14:05:23.613612+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":10,"log_dups_size":0,"ondisk_log_size":10,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T21:07:34.283014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"84'11","reported_seq":25,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.615353+0000","last_change":"2024-04-06T14:04:01.460563+0000","last_active":"2024-04-06T14:05:23.615353+0000","last_peered":"2024-04-06T14:05:23.615353+0000","last_clean":"2024-04-06T14:05:23.615353+0000","last_became_active":"2024-04-06T14:04:01.460249+0000","last_became_peered":"2024-04-06T14:04:01.460249+0000","last_unstale":"2024-04-06T14:05:23.615353+0000","last_undegraded":"2024-04-06T14:05:23.615353+0000","last_fullsized":"2024-04-06T14:05:23.615353+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T22:25:36.031752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"84'8","reported_seq":22,"reported_epoch":84,"state":"active+clean","last_fresh":"2024-04-06T14:05:23.598200+0000","last_change":"2024-04-06T14:04:01.454612+0000","last_active":"2024-04-06T14:05:23.598200+0000","last_peered":"2024-04-06T14:05:23.598200+0000","last_clean":"2024-04-06T14:05:23.598200+0000","last_became_active":"2024-04-06T14:04:01.454274+0000","last_became_peered":"2024-04-06T14:04:01.454274+0000","last_unstale":"2024-04-06T14:05:23.598200+0000","last_undegraded":"2024-04-06T14:05:23.598200+0000","last_fullsized":"2024-04-06T14:05:23.598200+0000","mapping_epoch":79,"log_start":"0'0","ondisk_log_start":"0'0","created":79,"last_epoch_clean":80,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-06T14:04:01.007186+0000","last_clean_scrub_stamp":"2024-04-06T14:04:01.007186+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-07T17:27:24.650594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":234,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":502,"ondisk_log_size":502,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":1771104,"data_compressed":107160,"data_compressed_allocated":135168,"data_compressed_original":1769472,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":597293,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":591,"num_read_kb":399,"num_write":269,"num_write_kb":1428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":40,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":565248,"data_stored":1791879,"data_compressed":81405,"data_compressed_allocated":110592,"data_compressed_original":1572864,"omap_allocated":0,"internal_metadata":0},"log_size":215,"ondisk_log_size":215,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":76,"seq":326417514523,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32000,"kb_used_data":884,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713152,"statfs":{"total":95995035648,"available":95962267648,"internally_reserved":0,"allocated":905216,"data_stored":444028,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.504},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.40500000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.375}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.745,"15min":0.745},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.58199999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.436},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55700000000000005}]},{"osd":9,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.55000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3149999999999999,"5min":1.3149999999999999,"15min":1.3149999999999999},"last":0.57499999999999996}]}]},{"osd":10,"up_from":68,"seq":292057776157,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28072,"kb_used_data":1052,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717080,"statfs":{"total":95995035648,"available":95966289920,"internally_reserved":0,"allocated":1077248,"data_stored":1235300,"data_compressed":46627,"data_compressed_allocated":61440,"data_compressed_original":729088,"omap_allocated":1599,"internal_metadata":27654593},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.61099999999999999,"15min":0.60699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.311},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.51600000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.74399999999999999,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.54600000000000004,"15min":0.54000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.52300000000000002,"15min":0.51500000000000001},"min":{"1min":0.40699999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71399999999999997,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.65700000000000003,"15min":0.65300000000000002},"min":{"1min":0.54600000000000004,"5min":0.501,"15min":0.501},"max":{"1min":0.84099999999999997,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67000000000000004,"15min":0.67300000000000004},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":0.84999999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.37}]},{"osd":6,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.68000000000000005,"15min":0.68799999999999994},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.75600000000000001,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.74099999999999999,"15min":0.71699999999999997},"min":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.46899999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.76400000000000001,"15min":0.76500000000000001},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.78500000000000003,"15min":0.79000000000000004},"min":{"1min":0.64900000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.92400000000000004,"5min":1.083,"15min":1.083},"last":0.51000000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":0.76000000000000001,"15min":0.76300000000000001},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":0.878,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76000000000000001,"15min":0.76600000000000001},"min":{"1min":0.58099999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.83199999999999996,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.53000000000000003}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":0.97999999999999998,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.498},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"min":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.54600000000000004}]}]},{"osd":9,"up_from":63,"seq":270582939681,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":924,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":946176,"data_stored":837480,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1598,"internal_metadata":27654594},"hb_peers":[0,1,2,3,4,6,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.40799999999999997,"15min":0.41599999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.371},{"interface":"front","average":{"1min":0.371,"5min":0.44900000000000001,"15min":0.46200000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.55600000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.314}]},{"osd":1,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57499999999999996,"15min":0.58099999999999996},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.753,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.75700000000000001,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.751,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.59599999999999997,"15min":0.60899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.74299999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.49399999999999999,"15min":0.50700000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.53400000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.49199999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.624,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.58899999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.69999999999999996,"5min":0.753,"15min":0.753},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.64800000000000002,"15min":0.66000000000000003},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88300000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.65600000000000003,"15min":0.67100000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.79400000000000004,"5min":1.107,"15min":1.107},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.63900000000000001,"15min":0.65200000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77800000000000002,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.57499999999999996,"15min":0.59699999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72499999999999998,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.61699999999999999,"15min":0.63400000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.73499999999999999,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.60599999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.68600000000000005,"15min":0.69599999999999995},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.90800000000000003,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.68999999999999995,"15min":0.70299999999999996},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.81399999999999995,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58599999999999997}]},{"osd":8,"last update":"Sat Apr 6 14:04:54 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.71199999999999997,"15min":0.72799999999999998},"min":{"1min":0.51100000000000001,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.76900000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.747,"15min":0.76400000000000001},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.83899999999999997,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.64400000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.66500000000000004,"15min":0.67000000000000004},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.82999999999999996,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.66600000000000004,"15min":0.66800000000000004},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65100000000000002}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]}]},{"osd":8,"up_from":56,"seq":240518168612,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27936,"kb_used_data":916,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717216,"statfs":{"total":95995035648,"available":95966429184,"internally_reserved":0,"allocated":937984,"data_stored":836487,"data_compressed":22064,"data_compressed_allocated":28672,"data_compressed_original":401408,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54900000000000004,"15min":0.55000000000000004},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.54900000000000004,"15min":0.55900000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60199999999999998,"15min":0.59799999999999998},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.755,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.379},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60899999999999999,"15min":0.60999999999999999},"min":{"1min":0.46800000000000003,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.76300000000000001,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.432}]},{"osd":2,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.46000000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.64700000000000002,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.45000000000000001,"15min":0.45200000000000001},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":0.77700000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40999999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79900000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71899999999999997,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45700000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.80100000000000005,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.33600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74399999999999999,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.38800000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.65000000000000002,"15min":0.65500000000000003},"min":{"1min":0.45500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80900000000000005,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.498},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.5,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71399999999999997,"15min":0.71599999999999997},"min":{"1min":0.504,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85899999999999999,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.69299999999999995,"15min":0.69099999999999995},"min":{"1min":0.56200000000000006,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.88,"5min":1.028,"15min":1.028},"last":0.51200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":0.71999999999999997,"15min":0.70999999999999996},"min":{"1min":0.628,"5min":0.501,"15min":0.501},"max":{"1min":1,"5min":1,"15min":1},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.69499999999999995,"15min":0.68400000000000005},"min":{"1min":0.53700000000000003,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.54000000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":0.70099999999999996,"15min":0.69299999999999995},"min":{"1min":0.59499999999999997,"5min":0.442,"15min":0.442},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.73699999999999999,"15min":0.73299999999999998},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.52600000000000002}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.55400000000000005}]}]},{"osd":7,"up_from":49,"seq":210453397543,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32060,"kb_used_data":944,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713092,"statfs":{"total":95995035648,"available":95962206208,"internally_reserved":0,"allocated":966656,"data_stored":1036537,"data_compressed":38638,"data_compressed_allocated":49152,"data_compressed_original":598016,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38900000000000001,"15min":0.373},"min":{"1min":0.313,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.58499999999999996,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.42399999999999999,"15min":0.40000000000000002},"min":{"1min":0.33800000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.66600000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.314}]},{"osd":2,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.45000000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.81899999999999995,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.57499999999999996,"15min":0.55900000000000005},"min":{"1min":0.40400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73599999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.53300000000000003,"15min":0.52200000000000002},"min":{"1min":0.42799999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.623},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.54700000000000004},"min":{"1min":0.41499999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.50900000000000001,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.75700000000000001,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.46600000000000003,"15min":0.46300000000000002},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64400000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.59999999999999998,"15min":0.58299999999999996},"min":{"1min":0.41499999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.623,"15min":0.60599999999999998},"min":{"1min":0.442,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.373}]},{"osd":6,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.63200000000000001,"15min":0.621},"min":{"1min":0.45700000000000002,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.84399999999999997,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.58499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78300000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.48699999999999999}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.66300000000000003,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.81899999999999995,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.44500000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:52 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.63400000000000001,"15min":0.63100000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.80800000000000005,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.46500000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.85499999999999998,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.47299999999999998}]},{"osd":10,"last update":"Sat Apr 6 14:05:09 2024","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":0.70099999999999996,"15min":0.69699999999999995},"min":{"1min":0.57199999999999995,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.67700000000000005,"15min":0.67300000000000004},"min":{"1min":0.52100000000000002,"5min":0.438,"15min":0.438},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.498}]},{"osd":11,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.504}]}]},{"osd":6,"up_from":45,"seq":193273528363,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32004,"kb_used_data":888,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713148,"statfs":{"total":95995035648,"available":95962263552,"internally_reserved":0,"allocated":909312,"data_stored":443653,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[0,1,2,3,4,5,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40699999999999997,"15min":0.41099999999999998},"min":{"1min":0.27400000000000002,"5min":0.245,"15min":0.245},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40000000000000002},"min":{"1min":0.28899999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.52400000000000002,"15min":0.50600000000000001},"min":{"1min":0.436,"5min":0.317,"15min":0.317},"max":{"1min":0.74299999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.54100000000000004,"15min":0.52700000000000002},"min":{"1min":0.42299999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.446,"5min":1.446,"15min":1.446},"last":0.44},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.57299999999999995,"15min":0.59099999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.70399999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44,"15min":0.44700000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.5269999999999999,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.51000000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.627},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.44900000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.61899999999999999,"15min":0.63},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.624,"15min":0.63400000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.78400000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Apr 6 14:04:22 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.65700000000000003,"15min":0.65500000000000003},"min":{"1min":0.48699999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.53600000000000003}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.495,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.79900000000000004,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.66100000000000003,"15min":0.65400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"last":0.497}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.63200000000000001,"15min":0.64400000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.365,"5min":1.365,"15min":1.365},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.71899999999999997,"5min":0.68700000000000006,"15min":0.68100000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.1379999999999999,"5min":4.1379999999999999,"15min":4.1379999999999999},"last":0.64300000000000002}]},{"osd":10,"last update":"Sat Apr 6 14:05:12 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.66900000000000004,"15min":0.65900000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.68799999999999994,"15min":0.68400000000000005},"min":{"1min":0.56100000000000005,"5min":0.503,"15min":0.503},"max":{"1min":1.6499999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.58799999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574909,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27968,"kb_used_data":952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717184,"statfs":{"total":95995035648,"available":95966396416,"internally_reserved":0,"allocated":974848,"data_stored":511168,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.47399999999999998,"15min":0.47199999999999998},"min":{"1min":0.35899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.48799999999999999,"15min":0.47299999999999998},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.84899999999999998,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.56000000000000005,"15min":0.51000000000000001},"min":{"1min":0.46000000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81499999999999995,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.55500000000000005,"15min":0.51700000000000002},"min":{"1min":0.47599999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77500000000000002,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.52600000000000002,"15min":0.49299999999999999},"min":{"1min":0.38900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53500000000000003,"15min":0.51300000000000001},"min":{"1min":0.46800000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69399999999999995,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Apr 6 14:04:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46100000000000002,"15min":0.44700000000000001},"min":{"1min":0.38500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.76500000000000001,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.41799999999999998},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66500000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:46 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.59599999999999997,"15min":0.57699999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.871,"5min":0.877,"15min":0.877},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.59099999999999997,"15min":0.56499999999999995},"min":{"1min":0.441,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.79900000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.60299999999999998,"15min":0.58999999999999997},"min":{"1min":0.48699999999999999,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81899999999999995,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.47299999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.86299999999999999,"5min":0.874,"15min":0.874},"last":0.66600000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:05:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.57199999999999995,"15min":0.55200000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.64000000000000001,"15min":0.62},"min":{"1min":0.501,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.875,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.64100000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.70099999999999996,"15min":0.69499999999999995},"min":{"1min":0.53400000000000003,"5min":0.503,"15min":0.503},"max":{"1min":0.90700000000000003,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.65700000000000003,"15min":0.65000000000000002},"min":{"1min":0.56499999999999995,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.83799999999999997,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.60699999999999998}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.68000000000000005,"15min":0.67600000000000005},"min":{"1min":0.5,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.86399999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.624},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.748,"15min":0.74299999999999999},"min":{"1min":0.56999999999999995,"5min":0.502,"15min":0.502},"max":{"1min":0.91400000000000003,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.70599999999999996}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.71399999999999997,"15min":0.71099999999999997},"min":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":0.88,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.72299999999999998,"15min":0.71699999999999997},"min":{"1min":0.56799999999999995,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":0.876,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.67600000000000005}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.65200000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27924,"kb_used_data":908,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717228,"statfs":{"total":95995035648,"available":95966441472,"internally_reserved":0,"allocated":929792,"data_stored":444924,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.49299999999999999,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.66400000000000003,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.498,"15min":0.50600000000000001},"min":{"1min":0.34599999999999997,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.624,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.57099999999999995,"15min":0.56699999999999995},"min":{"1min":0.44,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.70599999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.56899999999999995},"min":{"1min":0.42499999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.69199999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Apr 6 14:05:01 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.435,"5min":0.435,"15min":0.435},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.497,"15min":0.499},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.65100000000000002,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.375},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53400000000000003,"15min":0.54100000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.60899999999999999,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32600000000000001}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.54600000000000004,"15min":0.54400000000000004},"min":{"1min":0.45400000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55500000000000005,"15min":0.54500000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78700000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.44900000000000001,"15min":0.46300000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.499,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.432,"5min":0.46899999999999997,"15min":0.47499999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.55900000000000005,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Apr 6 14:03:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58699999999999997,"15min":0.58899999999999997},"min":{"1min":0.502,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.65500000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.64300000000000002,"15min":0.64700000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.58299999999999996}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.57999999999999996,"15min":0.59699999999999998},"min":{"1min":0.378,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.55100000000000005,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.54700000000000004,"15min":0.56100000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.627,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.432}]},{"osd":10,"last update":"Sat Apr 6 14:05:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58999999999999997,"15min":0.59199999999999997},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.68100000000000005,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.64100000000000001,"15min":0.64900000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.72299999999999998,"5min":1.0669999999999999,"15min":1.0669999999999999},"last":0.49399999999999999}]},{"osd":11,"last update":"Sat Apr 6 14:04:27 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56999999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411385,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":976,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":999424,"data_stored":512863,"data_compressed":4751,"data_compressed_allocated":8192,"data_compressed_original":16384,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.51300000000000001,"15min":0.46600000000000003},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.46899999999999997,"15min":0.443},"min":{"1min":0.33300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.72999999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.54100000000000004,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79300000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.56100000000000005,"15min":0.52600000000000002},"min":{"1min":0.45200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.79600000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.443}]},{"osd":3,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.58499999999999996,"15min":0.55500000000000005},"min":{"1min":0.45100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.391,"5min":3.391,"15min":3.391},"last":0.5},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.56499999999999995,"15min":0.54200000000000004},"min":{"1min":0.46500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":2.109,"5min":2.109,"15min":2.109},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:05:03 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.45600000000000002,"15min":0.436},"min":{"1min":0.307,"5min":0.216,"15min":0.216},"max":{"1min":0.78400000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.44400000000000001,"15min":0.42199999999999999},"min":{"1min":0.34999999999999998,"5min":0.214,"15min":0.214},"max":{"1min":0.66800000000000004,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.60999999999999999,"15min":0.60099999999999998},"min":{"1min":0.39500000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.82599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.59499999999999997,"15min":0.57999999999999996},"min":{"1min":0.51300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Apr 6 14:03:17 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.754},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.70799999999999996}]},{"osd":8,"last update":"Sat Apr 6 14:04:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.56499999999999995,"15min":0.56100000000000005},"min":{"1min":0.35999999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59999999999999998,"15min":0.60099999999999998},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.45200000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:55 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.71499999999999997,"15min":0.71099999999999997},"min":{"1min":0.57299999999999995,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.90600000000000003,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.66500000000000004,"15min":0.65500000000000003},"min":{"1min":0.46700000000000003,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53400000000000003}]},{"osd":10,"last update":"Sat Apr 6 14:05:16 2024","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.74199999999999999,"15min":0.72299999999999998},"min":{"1min":0.63100000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":2.488,"5min":2.488,"15min":2.488},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.74199999999999999,"15min":0.72699999999999998},"min":{"1min":0.61499999999999999,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.7799999999999998,"5min":2.7799999999999998,"15min":2.7799999999999998},"last":0.57599999999999996}]},{"osd":11,"last update":"Sat Apr 6 14:05:00 2024","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56000000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149750,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":36156,"kb_used_data":948,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93708996,"statfs":{"total":95995035648,"available":95958011904,"internally_reserved":0,"allocated":970752,"data_stored":968780,"data_compressed":28220,"data_compressed_allocated":36864,"data_compressed_original":524288,"omap_allocated":1592,"internal_metadata":36043208},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.314,"5min":0.33700000000000002,"15min":0.33300000000000002},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.42899999999999999,"5min":0.629,"15min":0.629},"last":0.314},{"interface":"front","average":{"1min":0.32300000000000001,"5min":0.34200000000000003,"15min":0.33200000000000002},"min":{"1min":0.253,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.41699999999999998,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.50900000000000001,"15min":0.48699999999999999},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.68999999999999995,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51300000000000001,"15min":0.49099999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.70599999999999996,"5min":0.872,"15min":0.872},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53700000000000003,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.66200000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.54000000000000004},"min":{"1min":0.436,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.63500000000000001,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.82099999999999995}]},{"osd":4,"last update":"Sat Apr 6 14:04:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.55100000000000005,"15min":0.54500000000000004},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.65200000000000002,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58899999999999997,"15min":0.57699999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.54700000000000004,"15min":0.53200000000000003},"min":{"1min":0.45500000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.72699999999999998,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.64300000000000002,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.82899999999999996}]},{"osd":6,"last update":"Sat Apr 6 14:05:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.47499999999999998,"15min":0.48499999999999999},"min":{"1min":0.32300000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.55600000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.47599999999999998,"15min":0.49399999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.54300000000000004,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Apr 6 14:05:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56100000000000005,"15min":0.56200000000000006},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.754,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.754},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58799999999999997,"15min":0.58899999999999997},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.76400000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.76400000000000001}]},{"osd":8,"last update":"Sat Apr 6 14:03:35 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.503}]},{"osd":9,"last update":"Sat Apr 6 14:04:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.60199999999999998,"15min":0.626},"min":{"1min":0.35599999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.53800000000000003,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.50700000000000001,"15min":0.52300000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.504,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.81000000000000005}]},{"osd":10,"last update":"Sat Apr 6 14:05:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.624,"15min":0.63600000000000001},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.72199999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.72999999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.79800000000000004}]},{"osd":11,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.497,"5min":0.497,"15min":0.497},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.83799999999999997}]}]},{"osd":4,"up_from":33,"seq":141733920819,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27916,"kb_used_data":896,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717236,"statfs":{"total":95995035648,"available":95966449664,"internally_reserved":0,"allocated":917504,"data_stored":445472,"data_compressed":2918,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:03:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.44400000000000001,"15min":0.434},"min":{"1min":0.311,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45400000000000001,"15min":0.45300000000000001},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.73999999999999999,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.375,"15min":0.35399999999999998},"min":{"1min":0.32000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.40200000000000002,"15min":0.378},"min":{"1min":0.30299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.70199999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.54600000000000004,"15min":0.52200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81399999999999995,"5min":1.036,"15min":1.036},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.55800000000000005,"15min":0.53600000000000003},"min":{"1min":0.379,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78900000000000003,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Apr 6 14:04:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.51900000000000002,"15min":0.50700000000000001},"min":{"1min":0.40400000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76500000000000001,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.441},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.47999999999999998,"15min":0.47199999999999998},"min":{"1min":0.376,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68500000000000005,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Apr 6 14:04:47 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.57199999999999995,"15min":0.55400000000000005},"min":{"1min":0.40600000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82299999999999995,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.628,"5min":0.60399999999999998,"15min":0.58799999999999997},"min":{"1min":0.46000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79600000000000004,"5min":1.012,"15min":1.012},"last":0.496}]},{"osd":6,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.56599999999999995,"15min":0.54200000000000004},"min":{"1min":0.45000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":0.72599999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.54500000000000004,"15min":0.51500000000000001},"min":{"1min":0.45900000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Apr 6 14:04:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.44900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.57699999999999996,"15min":0.56299999999999994},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.35699999999999998}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.68000000000000005,"15min":0.67700000000000005},"min":{"1min":0.52200000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85799999999999998,"5min":0.879,"15min":0.879},"last":0.432},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73899999999999999,"15min":0.746},"min":{"1min":0.57999999999999996,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.81999999999999995,"5min":1.694,"15min":1.694},"last":0.45400000000000001}]},{"osd":9,"last update":"Sat Apr 6 14:04:56 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.71599999999999997,"15min":0.71999999999999997},"min":{"1min":0.53300000000000003,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83299999999999996,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.62},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.69599999999999995,"15min":0.69199999999999995},"min":{"1min":0.54500000000000004,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.80400000000000005,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61099999999999999}]},{"osd":10,"last update":"Sat Apr 6 14:05:13 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.76900000000000002,"15min":0.77800000000000002},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":0.86199999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.71699999999999997,"5min":0.73199999999999998,"15min":0.73399999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.48199999999999998}]},{"osd":11,"last update":"Sat Apr 6 14:05:02 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.53500000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32092,"kb_used_data":980,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713060,"statfs":{"total":95995035648,"available":95962173440,"internally_reserved":0,"allocated":1003520,"data_stored":1167352,"data_compressed":44794,"data_compressed_allocated":57344,"data_compressed_original":720896,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.5,"15min":0.48299999999999998},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.70199999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.495,"15min":0.47899999999999998},"min":{"1min":0.36799999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.73099999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.53500000000000003,"15min":0.505},"min":{"1min":0.441,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.52500000000000002,"15min":0.497},"min":{"1min":0.44600000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.40200000000000002,"15min":0.376},"min":{"1min":0.30599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.65100000000000002,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.318},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.41799999999999998,"15min":0.38900000000000001},"min":{"1min":0.375,"5min":0.223,"15min":0.223},"max":{"1min":0.65700000000000003,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.52500000000000002,"15min":0.51800000000000002},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.67600000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.54300000000000004,"15min":0.52100000000000002},"min":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.74299999999999999,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Apr 6 14:04:45 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.59999999999999998,"15min":0.57099999999999995},"min":{"1min":0.438,"5min":0.432,"15min":0.432},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.58099999999999996,"15min":0.57099999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Apr 6 14:05:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59699999999999998,"15min":0.59199999999999997},"min":{"1min":0.45400000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.80300000000000005,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.624,"5min":0.629,"15min":0.61899999999999999},"min":{"1min":0.497,"5min":0.378,"15min":0.378},"max":{"1min":0.76400000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.435}]},{"osd":7,"last update":"Sat Apr 6 14:05:20 2024","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.64600000000000002,"15min":0.626},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.86199999999999999,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.64100000000000001,"15min":0.623},"min":{"1min":0.48899999999999999,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.85399999999999998,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.52600000000000002}]},{"osd":8,"last update":"Sat Apr 6 14:04:37 2024","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.625,"15min":0.61499999999999999},"min":{"1min":0.39200000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.68400000000000005,"15min":0.67600000000000005},"min":{"1min":0.50700000000000001,"5min":0.38,"15min":0.38},"max":{"1min":0.88200000000000001,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.48399999999999999}]},{"osd":9,"last update":"Sat Apr 6 14:04:53 2024","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.69499999999999995,"15min":0.69699999999999995},"min":{"1min":0.47499999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.84399999999999997,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.753,"15min":0.75800000000000001},"min":{"1min":0.54000000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.94699999999999995,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.54900000000000004}]},{"osd":10,"last update":"Sat Apr 6 14:05:10 2024","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.60799999999999998,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":0.90600000000000003,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.749,"5min":0.71299999999999997,"15min":0.70699999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.64500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":590368,"data_compressed":35720,"data_compressed_allocated":45056,"data_compressed_original":589824,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":73728,"data_stored":67722,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":69417,"data_compressed":1833,"data_compressed_allocated":4096,"data_compressed_original":8192,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":61440,"data_stored":525100,"data_compressed":25302,"data_compressed_allocated":32768,"data_compressed_original":516096,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2026,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":49152,"data_stored":133421,"data_compressed":6156,"data_compressed_allocated":8192,"data_compressed_original":122880,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":393372,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":394034,"data_compressed":19146,"data_compressed_allocated":24576,"data_compressed_original":393216,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":201700,"data_compressed":7989,"data_compressed_allocated":12288,"data_compressed_original":131072,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-06T14:05:29.548 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:29.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 -- 172.21.15.90:0/2434524110 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc3410b1d0 msgr2=0x7fcc3410d690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 --2- 172.21.15.90:0/2434524110 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc3410b1d0 0x7fcc3410d690 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 crypto rx=0x7fcc2c00b0a0 tx=0x7fcc2c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 -- 172.21.15.90:0/2434524110 shutdown_connections 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 --2- 172.21.15.90:0/2434524110 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc3410b1d0 0x7fcc3410d690 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 --2- 172.21.15.90:0/2434524110 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc34101db0 0x7fcc34102190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 --2- 172.21.15.90:0/2434524110 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc3410abb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 -- 172.21.15.90:0/2434524110 >> 172.21.15.90:0/2434524110 conn(0x7fcc340fb5f0 msgr2=0x7fcc340fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 -- 172.21.15.90:0/2434524110 shutdown_connections 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.706+0000 7fcc3b235700 1 -- 172.21.15.90:0/2434524110 wait complete. 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 Processor -- start 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 -- start start 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc34197650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc3410b1d0 0x7fcc3419b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc3419c030 con 0x7fcc34101db0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc3419c1a0 con 0x7fcc341026d0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc3b235700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc3419c360 con 0x7fcc3410b1d0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50332/0 (socket says 172.21.15.90:50332) 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 -- 172.21.15.90:0/3089029310 learned_addr learned my addr 172.21.15.90:0/3089029310 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc397d2700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc3410b1d0 0x7fcc3419b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc33fff700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc34197650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 -- 172.21.15.90:0/3089029310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc3410b1d0 msgr2=0x7fcc3419b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc3410b1d0 0x7fcc3419b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 -- 172.21.15.90:0/3089029310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 msgr2=0x7fcc34197650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc34197650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc38fd1700 1 -- 172.21.15.90:0/3089029310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc2c009d00 con 0x7fcc34101db0 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.707+0000 7fcc33fff700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc34197650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:29.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc38fd1700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7fcc2800c9b0 tx=0x7fcc2800cd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc28007b00 con 0x7fcc34101db0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcc28007c60 con 0x7fcc34101db0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc3419c640 con 0x7fcc34101db0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc28018660 con 0x7fcc34101db0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.708+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc341a3e80 con 0x7fcc34101db0 2024-04-06T14:05:29.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.709+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc340686e0 con 0x7fcc34101db0 2024-04-06T14:05:29.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.710+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fcc28018880 con 0x7fcc34101db0 2024-04-06T14:05:29.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.710+0000 7fcc31ffb700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 0x7fcc240a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:29.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.711+0000 7fcc33fff700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 0x7fcc240a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:29.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.711+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fcc280cd000 con 0x7fcc34101db0 2024-04-06T14:05:29.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.711+0000 7fcc33fff700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 0x7fcc240a3870 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fcc34198660 tx=0x7fcc20005e50 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:29.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.715+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fcc28096210 con 0x7fcc34101db0 2024-04-06T14:05:29.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.915+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcc34110b90 con 0x7fcc34101db0 2024-04-06T14:05:29.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.916+0000 7fcc31ffb700 1 -- 172.21.15.90:0/3089029310 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v84) v1 ==== 74+0+22920 (secure 0 0 0) 0x7fcc28099830 con 0x7fcc34101db0 2024-04-06T14:05:29.917 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:29.918 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":84,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:23.551387+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.918+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 msgr2=0x7fcc240a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.918+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 0x7fcc240a3870 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fcc34198660 tx=0x7fcc20005e50 comp rx=0 tx=0).stop 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.918+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 msgr2=0x7fcc34197110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.918+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 crypto rx=0x7fcc2800c9b0 tx=0x7fcc2800cd70 comp rx=0 tx=0).stop 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 shutdown_connections 2024-04-06T14:05:29.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcc240a13b0 0x7fcc240a3870 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcc3410b1d0 0x7fcc3419b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcc34101db0 0x7fcc34197110 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 --2- 172.21.15.90:0/3089029310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcc341026d0 0x7fcc34197650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 >> 172.21.15.90:0/3089029310 conn(0x7fcc340fb5f0 msgr2=0x7fcc340fcc60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 shutdown_connections 2024-04-06T14:05:29.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:29.919+0000 7fcc3b235700 1 -- 172.21.15.90:0/3089029310 wait complete. 2024-04-06T14:05:29.932 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '81', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:05:29.932 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.057+0000 7f86545f7700 1 -- 172.21.15.90:0/3009654622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 msgr2=0x7f864c069930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.057+0000 7f86545f7700 1 --2- 172.21.15.90:0/3009654622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c069930 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 crypto rx=0x7f86480099c0 tx=0x7f8648009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- 172.21.15.90:0/3009654622 shutdown_connections 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- 172.21.15.90:0/3009654622 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1060b0 0x7f864c10dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- 172.21.15.90:0/3009654622 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c069930 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- 172.21.15.90:0/3009654622 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c069f00 0x7f864c105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- 172.21.15.90:0/3009654622 >> 172.21.15.90:0/3009654622 conn(0x7f864c076d80 msgr2=0x7f864c077190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- 172.21.15.90:0/3009654622 shutdown_connections 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- 172.21.15.90:0/3009654622 wait complete. 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 Processor -- start 2024-04-06T14:05:30.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- start start 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 0x7f864c1001b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1006f0 0x7f864c1a5300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f864c1a58d0 con 0x7f864c069550 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f864c1a5a40 con 0x7f864c1006f0 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f86545f7700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f864c1a5c00 con 0x7f864c1060b0 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.058+0000 7f8652393700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50350/0 (socket says 172.21.15.90:50350) 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 -- 172.21.15.90:0/2710974314 learned_addr learned my addr 172.21.15.90:0/2710974314 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8651b92700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 0x7f864c1001b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652b94700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1006f0 0x7f864c1a5300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 -- 172.21.15.90:0/2710974314 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 msgr2=0x7f864c1001b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 0x7f864c1001b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 -- 172.21.15.90:0/2710974314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1006f0 msgr2=0x7f864c1a5300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1006f0 0x7f864c1a5300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 -- 172.21.15.90:0/2710974314 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86480095f0 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8651b92700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 0x7f864c1001b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f8652393700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f864800b580 tx=0x7f8648005e80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f864801d070 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f864800fc30 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f864c1a5e80 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8648022df0 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.059+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f864c1a6240 con 0x7f864c069550 2024-04-06T14:05:30.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.060+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f864c109280 con 0x7f864c069550 2024-04-06T14:05:30.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.062+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f8648022500 con 0x7f864c069550 2024-04-06T14:05:30.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.062+0000 7f86437fe700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 0x7f86380a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.062+0000 7f8651b92700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 0x7f86380a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.062+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f86480ce2d0 con 0x7f864c069550 2024-04-06T14:05:30.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.063+0000 7f8651b92700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 0x7f86380a3870 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f863c005950 tx=0x7f863c0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.065+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8648097620 con 0x7f864c069550 2024-04-06T14:05:30.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.259+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f864c102460 con 0x7f864c069550 2024-04-06T14:05:30.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.259+0000 7f86437fe700 1 -- 172.21.15.90:0/2710974314 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v84) v1 ==== 93+0+10 (secure 0 0 0) 0x7f864809ac40 con 0x7f864c069550 2024-04-06T14:05:30.260 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 msgr2=0x7f86380a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 0x7f86380a3870 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f863c005950 tx=0x7f863c0058e0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 msgr2=0x7f864c0ffc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 crypto rx=0x7f864800b580 tx=0x7f8648005e80 comp rx=0 tx=0).stop 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 shutdown_connections 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86380a13b0 0x7f86380a3870 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f864c1006f0 0x7f864c1a5300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f864c069550 0x7f864c0ffc70 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 --2- 172.21.15.90:0/2710974314 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f864c1060b0 0x7f864c1001b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 >> 172.21.15.90:0/2710974314 conn(0x7f864c076d80 msgr2=0x7f864c0ff3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 shutdown_connections 2024-04-06T14:05:30.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.261+0000 7f86545f7700 1 -- 172.21.15.90:0/2710974314 wait complete. 2024-04-06T14:05:30.274 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:05:30.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 -- 172.21.15.90:0/1125887431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58101920 msgr2=0x7fcf58105960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 --2- 172.21.15.90:0/1125887431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58101920 0x7fcf58105960 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 crypto rx=0x7fcf48009990 tx=0x7fcf48009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 -- 172.21.15.90:0/1125887431 shutdown_connections 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 --2- 172.21.15.90:0/1125887431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58106090 0x7fcf5810dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 --2- 172.21.15.90:0/1125887431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 0x7fcf58101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 --2- 172.21.15.90:0/1125887431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58101920 0x7fcf58105960 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 -- 172.21.15.90:0/1125887431 >> 172.21.15.90:0/1125887431 conn(0x7fcf580fc7f0 msgr2=0x7fcf580fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 -- 172.21.15.90:0/1125887431 shutdown_connections 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.399+0000 7fcf5c950700 1 -- 172.21.15.90:0/1125887431 wait complete. 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 Processor -- start 2024-04-06T14:05:30.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 -- start start 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 0x7fcf580730b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58073b30 0x7fcf581a53b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf581a5980 con 0x7fcf58106090 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf581a5af0 con 0x7fcf58073b30 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5c950700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf581a5cb0 con 0x7fcf58100f70 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50368/0 (socket says 172.21.15.90:50368) 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 -- 172.21.15.90:0/3317610116 learned_addr learned my addr 172.21.15.90:0/3317610116 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf5659c700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 0x7fcf580730b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf56d9d700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58073b30 0x7fcf581a53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 -- 172.21.15.90:0/3317610116 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 msgr2=0x7fcf580730b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 0x7fcf580730b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 -- 172.21.15.90:0/3317610116 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58073b30 msgr2=0x7fcf581a53b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58073b30 0x7fcf581a53b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.400+0000 7fcf55d9b700 1 -- 172.21.15.90:0/3317610116 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcf480095f0 con 0x7fcf58106090 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf55d9b700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7fcf480059d0 tx=0x7fcf48004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf4801d070 con 0x7fcf58106090 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcf581a5f30 con 0x7fcf58106090 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcf48022470 con 0x7fcf58106090 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf48017650 con 0x7fcf58106090 2024-04-06T14:05:30.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.401+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcf581a62f0 con 0x7fcf58106090 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.402+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcf58109360 con 0x7fcf58106090 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.403+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fcf480225e0 con 0x7fcf58106090 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.403+0000 7fcf477fe700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 0x7fcf400a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.403+0000 7fcf5659c700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 0x7fcf400a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.403+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fcf480ce960 con 0x7fcf58106090 2024-04-06T14:05:30.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.404+0000 7fcf5659c700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 0x7fcf400a3870 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fcf4c009c60 tx=0x7fcf4c009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.407+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fcf48097c00 con 0x7fcf58106090 2024-04-06T14:05:30.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.603+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fcf58074590 con 0x7fcf58106090 2024-04-06T14:05:30.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.603+0000 7fcf477fe700 1 -- 172.21.15.90:0/3317610116 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v84) v1 ==== 104+0+11 (secure 0 0 0) 0x7fcf4809b220 con 0x7fcf58106090 2024-04-06T14:05:30.604 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:05:30.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.605+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 msgr2=0x7fcf400a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.605+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 0x7fcf400a3870 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fcf4c009c60 tx=0x7fcf4c009260 comp rx=0 tx=0).stop 2024-04-06T14:05:30.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.605+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 msgr2=0x7fcf580735f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.605+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 crypto rx=0x7fcf480059d0 tx=0x7fcf48004730 comp rx=0 tx=0).stop 2024-04-06T14:05:30.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 shutdown_connections 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcf400a13b0 0x7fcf400a3870 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcf58073b30 0x7fcf581a53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcf58100f70 0x7fcf580730b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 --2- 172.21.15.90:0/3317610116 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcf58106090 0x7fcf580735f0 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 >> 172.21.15.90:0/3317610116 conn(0x7fcf580fc7f0 msgr2=0x7fcf580feb60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 shutdown_connections 2024-04-06T14:05:30.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.606+0000 7fcf5c950700 1 -- 172.21.15.90:0/3317610116 wait complete. 2024-04-06T14:05:30.618 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:05:30.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.743+0000 7f203c91c700 1 -- 172.21.15.90:0/1276658573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20380ffe70 msgr2=0x7f2038100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.743+0000 7f203c91c700 1 --2- 172.21.15.90:0/1276658573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20380ffe70 0x7f2038100250 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 crypto rx=0x7f2028009990 tx=0x7f2028009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 -- 172.21.15.90:0/1276658573 shutdown_connections 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 --2- 172.21.15.90:0/1276658573 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2038104f70 0x7f203810caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 --2- 172.21.15.90:0/1276658573 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f20380ffe70 0x7f2038100250 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 --2- 172.21.15.90:0/1276658573 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038100820 0x7f2038104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 -- 172.21.15.90:0/1276658573 >> 172.21.15.90:0/1276658573 conn(0x7f2038076d50 msgr2=0x7f2038077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 -- 172.21.15.90:0/1276658573 shutdown_connections 2024-04-06T14:05:30.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.744+0000 7f203c91c700 1 -- 172.21.15.90:0/1276658573 wait complete. 2024-04-06T14:05:30.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 Processor -- start 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 -- start start 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 0x7f20381960c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038104f70 0x7f2038196600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203819b420 con 0x7f203819a9a0 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203819b590 con 0x7f20380ffe70 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203c91c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f203819b750 con 0x7f2038104f70 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2035d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038104f70 0x7f2038196600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50390/0 (socket says 172.21.15.90:50390) 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 -- 172.21.15.90:0/2256224738 learned_addr learned my addr 172.21.15.90:0/2256224738 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203659c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 0x7f20381960c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 -- 172.21.15.90:0/2256224738 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038104f70 msgr2=0x7f2038196600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038104f70 0x7f2038196600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 -- 172.21.15.90:0/2256224738 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 msgr2=0x7f20381960c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 0x7f20381960c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f2036d9d700 1 -- 172.21.15.90:0/2256224738 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20280095f0 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.745+0000 7f203659c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 0x7f20381960c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f2036d9d700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f2030007ee0 tx=0x7f203000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2030017070 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f203000f040 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20381a2cb0 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2030005740 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.746+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20381a31e0 con 0x7f203819a9a0 2024-04-06T14:05:30.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.747+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20381081e0 con 0x7f203819a9a0 2024-04-06T14:05:30.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.748+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f203000dc60 con 0x7f203819a9a0 2024-04-06T14:05:30.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.748+0000 7f20277fe700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 0x7f20200a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:30.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.748+0000 7f203659c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 0x7f20200a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:30.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.748+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f20300cc660 con 0x7f203819a9a0 2024-04-06T14:05:30.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.749+0000 7f203659c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 0x7f20200a38f0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f2028006010 tx=0x7f202800b540 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:30.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.753+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f2030095870 con 0x7f203819a9a0 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3089029310' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:30 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2710974314' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:30.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.950+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f20381973c0 con 0x7f203819a9a0 2024-04-06T14:05:30.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.950+0000 7f20277fe700 1 -- 172.21.15.90:0/2256224738 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v84) v1 ==== 100+0+11 (secure 0 0 0) 0x7f2030098e90 con 0x7f203819a9a0 2024-04-06T14:05:30.951 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 msgr2=0x7f20200a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 0x7f20200a38f0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f2028006010 tx=0x7f202800b540 comp rx=0 tx=0).stop 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 msgr2=0x7f203819ae20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 crypto rx=0x7f2030007ee0 tx=0x7f203000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 shutdown_connections 2024-04-06T14:05:30.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f20200a1430 0x7f20200a38f0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f203819a9a0 0x7f203819ae20 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f20380ffe70 0x7f20381960c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 --2- 172.21.15.90:0/2256224738 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2038104f70 0x7f2038196600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.952+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 >> 172.21.15.90:0/2256224738 conn(0x7f2038076d50 msgr2=0x7f20380fdea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.953+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 shutdown_connections 2024-04-06T14:05:30.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:30.953+0000 7f203c91c700 1 -- 172.21.15.90:0/2256224738 wait complete. 2024-04-06T14:05:30.964 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.964 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.964 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.965 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:30.965 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3089029310' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:30.965 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:30 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2710974314' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:30.966 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:05:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: from='client.15423 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-06T14:05:30.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: pgmap v6: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:05:30.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3089029310' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:30.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:30 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2710974314' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 -- 172.21.15.90:0/3315009369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc300ffae0 msgr2=0x7efc300fff60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 --2- 172.21.15.90:0/3315009369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc300ffae0 0x7efc300fff60 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 crypto rx=0x7efc24009990 tx=0x7efc24009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 -- 172.21.15.90:0/3315009369 shutdown_connections 2024-04-06T14:05:31.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 --2- 172.21.15.90:0/3315009369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc301004a0 0x7efc3010c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 --2- 172.21.15.90:0/3315009369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc30105f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 --2- 172.21.15.90:0/3315009369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc300ffae0 0x7efc300fff60 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.087+0000 7efc3503e700 1 -- 172.21.15.90:0/3315009369 >> 172.21.15.90:0/3315009369 conn(0x7efc300fb5f0 msgr2=0x7efc300fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.088+0000 7efc3503e700 1 -- 172.21.15.90:0/3315009369 shutdown_connections 2024-04-06T14:05:31.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.088+0000 7efc3503e700 1 -- 172.21.15.90:0/3315009369 wait complete. 2024-04-06T14:05:31.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.088+0000 7efc3503e700 1 Processor -- start 2024-04-06T14:05:31.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 -- start start 2024-04-06T14:05:31.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc300ffae0 0x7efc30195f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc3019a860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc3019aea0 con 0x7efc301004a0 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc3019b010 con 0x7efc30105b80 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc3503e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc3019b1d0 con 0x7efc300ffae0 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50414/0 (socket says 172.21.15.90:50414) 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 -- 172.21.15.90:0/1129747605 learned_addr learned my addr 172.21.15.90:0/1129747605 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2ed9d700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc300ffae0 0x7efc30195f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2f59e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc3019a860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 -- 172.21.15.90:0/1129747605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc300ffae0 msgr2=0x7efc30195f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc300ffae0 0x7efc30195f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 -- 172.21.15.90:0/1129747605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 msgr2=0x7efc3019a860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc3019a860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2e59c700 1 -- 172.21.15.90:0/1129747605 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc240095f0 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.089+0000 7efc2f59e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc3019a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc2e59c700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7efc240098e0 tx=0x7efc2400f670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc2401d070 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc3019b450 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc24022470 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc24017650 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.090+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc301a2cf0 con 0x7efc301004a0 2024-04-06T14:05:31.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.091+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc30108100 con 0x7efc301004a0 2024-04-06T14:05:31.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.092+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7efc240225e0 con 0x7efc301004a0 2024-04-06T14:05:31.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.092+0000 7efc17fff700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 0x7efc180a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.092+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7efc240ce330 con 0x7efc301004a0 2024-04-06T14:05:31.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.092+0000 7efc2ed9d700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 0x7efc180a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.093+0000 7efc2ed9d700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 0x7efc180a3870 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7efc20005950 tx=0x7efc2000b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.096+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efc240975d0 con 0x7efc301004a0 2024-04-06T14:05:31.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.343+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7efc30067a50 con 0x7efc301004a0 2024-04-06T14:05:31.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.343+0000 7efc17fff700 1 -- 172.21.15.90:0/1129747605 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7efc2409abf0 con 0x7efc301004a0 2024-04-06T14:05:31.344 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:31.344 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:05:31.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.345+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 msgr2=0x7efc180a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.345+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 0x7efc180a3870 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7efc20005950 tx=0x7efc2000b410 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 msgr2=0x7efc301964c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 crypto rx=0x7efc240098e0 tx=0x7efc2400f670 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 shutdown_connections 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efc180a13b0 0x7efc180a3870 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc30105b80 0x7efc3019a860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc300ffae0 0x7efc30195f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 --2- 172.21.15.90:0/1129747605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc301004a0 0x7efc301964c0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 >> 172.21.15.90:0/1129747605 conn(0x7efc300fb5f0 msgr2=0x7efc300fbe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 shutdown_connections 2024-04-06T14:05:31.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.346+0000 7efc3503e700 1 -- 172.21.15.90:0/1129747605 wait complete. 2024-04-06T14:05:31.348 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:05:31.358 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 -- 172.21.15.90:0/1037749372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 msgr2=0x7f78e80ff790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 --2- 172.21.15.90:0/1037749372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e80ff790 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 crypto rx=0x7f78e4009990 tx=0x7f78e4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 -- 172.21.15.90:0/1037749372 shutdown_connections 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 --2- 172.21.15.90:0/1037749372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e8105f70 0x7f78e810db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 --2- 172.21.15.90:0/1037749372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e80ff790 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 --2- 172.21.15.90:0/1037749372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e80ffd60 0x7f78e8105890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 -- 172.21.15.90:0/1037749372 >> 172.21.15.90:0/1037749372 conn(0x7f78e80760a0 msgr2=0x7f78e80fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 -- 172.21.15.90:0/1037749372 shutdown_connections 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.479+0000 7f78f00c9700 1 -- 172.21.15.90:0/1037749372 wait complete. 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 Processor -- start 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 -- start start 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e8105f70 0x7f78e81976a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e819ba40 0x7f78e819bec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78e819c4c0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78e819c630 con 0x7f78e8105f70 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78f00c9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78e819c7f0 con 0x7f78e819ba40 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78ede65700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78ede65700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50422/0 (socket says 172.21.15.90:50422) 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78ede65700 1 -- 172.21.15.90:0/962828173 learned_addr learned my addr 172.21.15.90:0/962828173 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78ee666700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e819ba40 0x7f78e819bec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.480+0000 7f78ed664700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e8105f70 0x7f78e81976a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 -- 172.21.15.90:0/962828173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e819ba40 msgr2=0x7f78e819bec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e819ba40 0x7f78e819bec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 -- 172.21.15.90:0/962828173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e8105f70 msgr2=0x7f78e81976a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e8105f70 0x7f78e81976a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 -- 172.21.15.90:0/962828173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78e40095f0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78ede65700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f78e40095c0 tx=0x7f78e400f740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78e401d070 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f78e40054f0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78e4004c40 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78e81a3cf0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.481+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78e81a40c0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.482+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78e8109200 con 0x7f78e80ff3b0 2024-04-06T14:05:31.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.483+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f78e400fc30 con 0x7f78e80ff3b0 2024-04-06T14:05:31.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.483+0000 7f78deffd700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 0x7f78d40a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.484+0000 7f78ed664700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 0x7f78d40a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.484+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f78e40ce1a0 con 0x7f78e80ff3b0 2024-04-06T14:05:31.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.484+0000 7f78ed664700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 0x7f78d40a3870 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f78d8005fd0 tx=0x7f78d800a560 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.487+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f78e4097400 con 0x7f78e80ff3b0 2024-04-06T14:05:31.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.682+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f78e8102460 con 0x7f78e80ff3b0 2024-04-06T14:05:31.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.684+0000 7f78deffd700 1 -- 172.21.15.90:0/962828173 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v84) v1 ==== 74+0+22920 (secure 0 0 0) 0x7f78e409aa20 con 0x7f78e80ff3b0 2024-04-06T14:05:31.685 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:31.685 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":84,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:05:23.551387+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"81","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 msgr2=0x7f78d40a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 0x7f78d40a3870 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f78d8005fd0 tx=0x7f78d800a560 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 msgr2=0x7f78e8197160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 crypto rx=0x7f78e40095c0 tx=0x7f78e400f740 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 shutdown_connections 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78d40a13b0 0x7f78d40a3870 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78e819ba40 0x7f78e819bec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78e80ff3b0 0x7f78e8197160 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 --2- 172.21.15.90:0/962828173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78e8105f70 0x7f78e81976a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 >> 172.21.15.90:0/962828173 conn(0x7f78e80760a0 msgr2=0x7f78e81042a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 shutdown_connections 2024-04-06T14:05:31.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.686+0000 7f78f00c9700 1 -- 172.21.15.90:0/962828173 wait complete. 2024-04-06T14:05:31.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3317610116' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2256224738' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.698 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1129747605' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:31.699 DEBUG:tasks.fwd_scrub:Yielding 2024-04-06T14:05:31.700 INFO:teuthology.run_tasks:Running task exec... 2024-04-06T14:05:31.709 INFO:teuthology.task.exec:Executing custom commands... 2024-04-06T14:05:31.710 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:05:31.710 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1m' 2024-04-06T14:05:31.710 INFO:tasks.fwd_scrub.fs.[cephfs]:start scrubbing fs: cephfs 2024-04-06T14:05:31.711 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:05:31.711 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:05:31.711 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:05:31.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 -- 172.21.15.90:0/720567510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff19810d700 msgr2=0x7ff19810faf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/720567510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff19810d700 0x7ff19810faf0 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 crypto rx=0x7ff19000b0a0 tx=0x7ff19000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 -- 172.21.15.90:0/720567510 shutdown_connections 2024-04-06T14:05:31.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/720567510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff19810d700 0x7ff19810faf0 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/720567510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff198100a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2227975639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec104f70 msgr2=0x7f86ec10caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2227975639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec104f70 0x7f86ec10caf0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 crypto rx=0x7f86e4009f90 tx=0x7f86e400b330 comp rx=0 tx=0).stop 2024-04-06T14:05:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/720567510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff198100fb0 0x7ff19810d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 -- 172.21.15.90:0/720567510 >> 172.21.15.90:0/720567510 conn(0x7ff198076d20 msgr2=0x7ff198077130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2227975639 shutdown_connections 2024-04-06T14:05:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2227975639 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec104f70 0x7f86ec10caf0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2227975639 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec0ffe70 0x7f86ec100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2227975639 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec100820 0x7f86ec104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 -- 172.21.15.90:0/720567510 shutdown_connections 2024-04-06T14:05:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2227975639 >> 172.21.15.90:0/2227975639 conn(0x7f86ec076d50 msgr2=0x7f86ec077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7ff19fbf4700 1 -- 172.21.15.90:0/720567510 wait complete. 2024-04-06T14:05:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2227975639 shutdown_connections 2024-04-06T14:05:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.838+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2227975639 wait complete. 2024-04-06T14:05:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 Processor -- start 2024-04-06T14:05:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 -- start start 2024-04-06T14:05:31.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7f86f0bfd700 1 Processor -- start 2024-04-06T14:05:31.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7f86f0bfd700 1 -- start start 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff1981994f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff19810d700 0x7ff19819ddd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff19819e410 con 0x7ff198100fb0 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff19819e580 con 0x7ff198100690 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19fbf4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff19819e740 con 0x7ff19810d700 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19d18f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19d18f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50452/0 (socket says 172.21.15.90:50452) 2024-04-06T14:05:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7ff19d18f700 1 -- 172.21.15.90:0/574635654 learned_addr learned my addr 172.21.15.90:0/574635654 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.839+0000 7f86f0bfd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86f0bfd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec104f70 0x7f86ec197870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86f0bfd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 0x7f86ec19c090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86f0bfd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86ec19c690 con 0x7f86ec19bc10 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86f0bfd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86ec19c800 con 0x7f86ec0ffe70 2024-04-06T14:05:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86f0bfd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f86ec19c9c0 con 0x7f86ec104f70 2024-04-06T14:05:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d990700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff1981994f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19e191700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff19810d700 0x7ff19819ddd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 -- 172.21.15.90:0/574635654 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff19810d700 msgr2=0x7ff19819ddd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff19810d700 0x7ff19819ddd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 -- 172.21.15.90:0/574635654 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 msgr2=0x7ff1981994f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff1981994f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:40352/0 (socket says 172.21.15.90:40352) 2024-04-06T14:05:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 -- 172.21.15.90:0/2132938353 learned_addr learned my addr 172.21.15.90:0/2132938353 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 -- 172.21.15.90:0/574635654 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff190009d00 con 0x7ff198100fb0 2024-04-06T14:05:31.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec104f70 0x7f86ec197870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 0x7f86ec19c090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d990700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff1981994f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:31.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19d18f700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7ff18800d980 tx=0x7ff18800dd40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 -- 172.21.15.90:0/2132938353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec104f70 msgr2=0x7f86ec197870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec104f70 0x7f86ec197870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 -- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 msgr2=0x7f86ec19c090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 0x7f86ec19c090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 -- 172.21.15.90:0/2132938353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f86e4009c40 con 0x7f86ec0ffe70 2024-04-06T14:05:31.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 0x7f86ec19c090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:31.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff18800de60 con 0x7ff198100fb0 2024-04-06T14:05:31.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1880044e0 con 0x7ff198100fb0 2024-04-06T14:05:31.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff188003be0 con 0x7ff198100fb0 2024-04-06T14:05:31.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff19819ea20 con 0x7ff198100fb0 2024-04-06T14:05:31.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f86dc00d950 tx=0x7f86dc00dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1981a6260 con 0x7ff198100fb0 2024-04-06T14:05:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.840+0000 7f86db7fe700 1 -- 172.21.15.90:0/2132938353 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc009930 con 0x7f86ec0ffe70 2024-04-06T14:05:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.841+0000 7f86db7fe700 1 -- 172.21.15.90:0/2132938353 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f86dc010460 con 0x7f86ec0ffe70 2024-04-06T14:05:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.841+0000 7f86db7fe700 1 -- 172.21.15.90:0/2132938353 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc00f620 con 0x7f86ec0ffe70 2024-04-06T14:05:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.841+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f86ec1a3f20 con 0x7f86ec0ffe70 2024-04-06T14:05:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.841+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86ec1a42f0 con 0x7f86ec0ffe70 2024-04-06T14:05:31.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.841+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff198108920 con 0x7ff198100fb0 2024-04-06T14:05:31.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.842+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7ff188009bb0 con 0x7ff198100fb0 2024-04-06T14:05:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.843+0000 7ff18effd700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 0x7ff1840a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.843+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff1880ccf90 con 0x7ff198100fb0 2024-04-06T14:05:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.843+0000 7ff19d990700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 0x7ff1840a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.844+0000 7ff19d990700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 0x7ff1840a38f0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ff194009cb0 tx=0x7ff194009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.844+0000 7f86db7fe700 1 -- 172.21.15.90:0/2132938353 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f86dc0108d0 con 0x7f86ec0ffe70 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.844+0000 7f86db7fe700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 0x7f86d40a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.844+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 0x7f86d40a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.844+0000 7f86db7fe700 1 -- 172.21.15.90:0/2132938353 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f86dc0cccc0 con 0x7f86ec0ffe70 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.845+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 0x7f86d40a3870 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f86ec198880 tx=0x7f86e0009310 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 Processor -- start 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 -- start start 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24df800 0x55bee24dfbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee24ec250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee2437a10 con 0x55bee24e0120 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee24353d0 con 0x55bee24df800 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee2430cc0 con 0x55bee24e9e60 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50456/0 (socket says 172.21.15.90:50456) 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.846+0000 7f86e9d9b700 1 -- 172.21.15.90:0/3948291927 learned_addr learned my addr 172.21.15.90:0/3948291927 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86ea59c700 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24df800 0x55bee24dfbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86ead9d700 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee24ec250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 -- 172.21.15.90:0/3948291927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 msgr2=0x55bee24ec250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee24ec250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 -- 172.21.15.90:0/3948291927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24df800 msgr2=0x55bee24dfbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24df800 0x55bee24dfbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 -- 172.21.15.90:0/3948291927 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bee22b2430 con 0x55bee24e0120 2024-04-06T14:05:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.847+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f86e0011d20 tx=0x7f86e0014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29d8c2d4045b7b81 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.848+0000 7f86d8ff9700 1 -- 172.21.15.90:0/3948291927 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86e00098d0 con 0x55bee24e0120 2024-04-06T14:05:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86d8ff9700 1 -- 172.21.15.90:0/3948291927 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f86e0005020 con 0x55bee24e0120 2024-04-06T14:05:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86d8ff9700 1 -- 172.21.15.90:0/3948291927 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86e00039f0 con 0x55bee24e0120 2024-04-06T14:05:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff188096220 con 0x7ff198100fb0 2024-04-06T14:05:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/3948291927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 msgr2=0x55bee24e9920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 crypto rx=0x7f86e0011d20 tx=0x7f86e0014000 comp rx=0 tx=0).stop 2024-04-06T14:05:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/3948291927 shutdown_connections 2024-04-06T14:05:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee24ec250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24df800 0x55bee24dfbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/3948291927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e0120 0x55bee24e9920 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.849+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/3948291927 >> 172.21.15.90:0/3948291927 conn(0x55bee243e720 msgr2=0x55bee24dab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/3948291927 shutdown_connections 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/3948291927 wait complete. 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- start start 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 Processor -- start 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24df800 0x55bee24e6c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee24e71b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee23cce00 con 0x55bee24e9e60 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee23ccaa0 con 0x55bee24e0120 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee22b4720 con 0x55bee24df800 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86fcdb1b80 10 client.0 _ll_register_callbacks cb 0x55bee23d64f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86ead9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86ead9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50464/0 (socket says 172.21.15.90:50464) 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.850+0000 7f86ead9d700 1 -- 172.21.15.90:0/2763390072 learned_addr learned my addr 172.21.15.90:0/2763390072 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24df800 0x55bee24e6c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee24e71b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 -- 172.21.15.90:0/2763390072 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24df800 msgr2=0x55bee24e6c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24df800 0x55bee24e6c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 -- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 msgr2=0x55bee24e71b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee24e71b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 -- 172.21.15.90:0/2763390072 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86e0017070 con 0x55bee24e9e60 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee24e71b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f86e4003c30 tx=0x7f86e4007f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2c1ca2b7678f78e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86e400e040 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f86e4003e40 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bee23f1400 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f86e4009420 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86fcdb1b80 10 client.15477 fetch_fsmap learned FSMap version 24 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bee225ebe0 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55bee23f1400 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.851+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f86e400b050 con 0x55bee24e9e60 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 10 client.15477 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 10 client.15477 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 20 client.15477 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee24f9aa0 0x55bee24fbec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 4 client.15477 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bee2429780 con 0x55bee24f9aa0 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee24f9aa0 0x55bee24fbec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee24f9aa0 0x55bee24fbec0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6e13f26a06d209cc server_cookie=48e0a8fa876a5943 in_seq=0 out_seq=0 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.852+0000 7f86d1ffb700 10 client.15477 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2763390072 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55bee2429780 con 0x55bee24f9aa0 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86d1ffb700 10 client.15477 handle_command_reply: tid=0 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 client.15477 shutdown 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 msgr2=0x55bee24e76f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 crypto rx=0x7f86e4003c30 tx=0x7f86e4007f90 comp rx=0 tx=0).stop 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 shutdown_connections 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24e9e60 0x55bee24e76f0 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee24f9aa0 0x55bee24fbec0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24df800 0x55bee24e6c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2763390072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee24e71b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 >> 172.21.15.90:0/2763390072 conn(0x55bee243e720 msgr2=0x55bee24d9ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86d1ffb700 0 client.15477 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 shutdown_connections 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2763390072 wait complete. 2024-04-06T14:05:31.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.853+0000 7f86fcdb1b80 20 client.15477 trim_cache size 0 max 16384 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 Processor -- start 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 -- start start 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e0120 0x55bee24def00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 0x55bee24e3ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee24d2420 con 0x55bee24df800 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee24f1020 con 0x55bee24e9e60 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.870+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee24e5180 con 0x55bee24e0120 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50478/0 (socket says 172.21.15.90:50478) 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 -- 172.21.15.90:0/939094821 learned_addr learned my addr 172.21.15.90:0/939094821 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e0120 0x55bee24def00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ead9d700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 0x55bee24e3ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 -- 172.21.15.90:0/939094821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e0120 msgr2=0x55bee24def00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e0120 0x55bee24def00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 -- 172.21.15.90:0/939094821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 msgr2=0x55bee24e3ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 0x55bee24e3ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 -- 172.21.15.90:0/939094821 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bee24d6be0 con 0x55bee24df800 2024-04-06T14:05:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ead9d700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 0x55bee24e3ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86ea59c700 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f86dc099b20 tx=0x7f86dc010cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0ab0bdc7885b396 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86d97fa700 1 -- 172.21.15.90:0/939094821 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc0b1070 con 0x55bee24df800 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86d97fa700 1 -- 172.21.15.90:0/939094821 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f86dc09d9c0 con 0x55bee24df800 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.871+0000 7f86d97fa700 1 -- 172.21.15.90:0/939094821 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc014070 con 0x55bee24df800 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/939094821 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 msgr2=0x55bee24f0ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 crypto rx=0x7f86dc099b20 tx=0x7f86dc010cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/939094821 shutdown_connections 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e9e60 0x55bee24e3ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee24f0ae0 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/939094821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e0120 0x55bee24def00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/939094821 >> 172.21.15.90:0/939094821 conn(0x55bee243e720 msgr2=0x55bee24de7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/939094821 shutdown_connections 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/939094821 wait complete. 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 -- start start 2024-04-06T14:05:31.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.872+0000 7f86fcdb1b80 1 Processor -- start 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee254c0f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee25460f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee25466d0 con 0x55bee24df800 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee2546840 con 0x55bee24e0120 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bee2546a00 con 0x55bee24e9e60 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 10 client.0 _ll_register_callbacks cb 0x55bee23d64f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50480/0 (socket says 172.21.15.90:50480) 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 -- 172.21.15.90:0/2180480698 learned_addr learned my addr 172.21.15.90:0/2180480698 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ead9d700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee25460f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee254c0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 -- 172.21.15.90:0/2180480698 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 msgr2=0x55bee25460f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee25460f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 -- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 msgr2=0x55bee254c0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee254c0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 -- 172.21.15.90:0/2180480698 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f86ec1a42f0 con 0x55bee24df800 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86ea59c700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f86dc010fd0 tx=0x7f86dc0a2080 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d14ef7f1e9b7e924 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f86dc0b1070 con 0x55bee24df800 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f86dc099df0 con 0x55bee24df800 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.873+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f86e0005020 con 0x55bee24df800 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f86dc014040 con 0x55bee24df800 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 10 client.15483 fetch_fsmap learned FSMap version 24 2024-04-06T14:05:31.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bee23f6f20 con 0x55bee24df800 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f86e0005020 con 0x55bee24df800 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f86dc0a1b80 con 0x55bee24df800 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 10 client.15483 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 10 client.15483 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 20 client.15483 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee25535d0 0x55bee25559c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 4 client.15483 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55bee227fd70 con 0x55bee25535d0 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.874+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee25535d0 0x55bee25559c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.875+0000 7f86d1ffb700 10 client.15483 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.875+0000 7f86e9d9b700 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee25535d0 0x55bee25559c0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c64755edd1b524d server_cookie=6b89765ded5007b9 in_seq=0 out_seq=0 2024-04-06T14:05:31.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86d1ffb700 1 -- 172.21.15.90:0/2180480698 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55bee227fd70 con 0x55bee25535d0 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86d1ffb700 10 client.15483 handle_command_reply: tid=0 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 client.15483 shutdown 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 msgr2=0x55bee254bbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 crypto rx=0x7f86dc010fd0 tx=0x7f86dc0a2080 comp rx=0 tx=0).stop 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 shutdown_connections 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55bee25535d0 0x55bee25559c0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bee24e9e60 0x55bee25460f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86d1ffb700 0 client.15483 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bee24df800 0x55bee254bbb0 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 --2- 172.21.15.90:0/2180480698 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bee24e0120 0x55bee254c0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 >> 172.21.15.90:0/2180480698 conn(0x55bee243e720 msgr2=0x55bee24df080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 shutdown_connections 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 1 -- 172.21.15.90:0/2180480698 wait complete. 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.876+0000 7f86fcdb1b80 20 client.15483 trim_cache size 0 max 16384 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "39b3d254-51ce-4d50-a022-eb7f865a85a8", 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:05:31.884 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:05:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 msgr2=0x7f86d40a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 0x7f86d40a3870 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f86ec198880 tx=0x7f86e0009310 comp rx=0 tx=0).stop 2024-04-06T14:05:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 msgr2=0x7f86ec197330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f86dc00d950 tx=0x7f86dc00dc60 comp rx=0 tx=0).stop 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 shutdown_connections 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f86d40a13b0 0x7f86d40a3870 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f86ec19bc10 0x7f86ec19c090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f86ec0ffe70 0x7f86ec197330 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 --2- 172.21.15.90:0/2132938353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f86ec104f70 0x7f86ec197870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 >> 172.21.15.90:0/2132938353 conn(0x7f86ec076d50 msgr2=0x7f86ec0fdcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 shutdown_connections 2024-04-06T14:05:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:31.877+0000 7f86f0bfd700 1 -- 172.21.15.90:0/2132938353 wait complete. 2024-04-06T14:05:31.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3317610116' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2256224738' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1129747605' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:31.893 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:05:31.894 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:05:31.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3317610116' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2256224738' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:05:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1129747605' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:05:32.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- 172.21.15.90:0/2633850754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c4068aa0 msgr2=0x7fb7c4105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 --2- 172.21.15.90:0/2633850754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c4068aa0 0x7fb7c4105970 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 crypto rx=0x7fb7b40099c0 tx=0x7fb7b4009cd0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- 172.21.15.90:0/2633850754 shutdown_connections 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 --2- 172.21.15.90:0/2633850754 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41060a0 0x7fb7c410dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 --2- 172.21.15.90:0/2633850754 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 0x7fb7c40684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 --2- 172.21.15.90:0/2633850754 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c4068aa0 0x7fb7c4105970 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- 172.21.15.90:0/2633850754 >> 172.21.15.90:0/2633850754 conn(0x7fb7c4076d50 msgr2=0x7fb7c4077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- 172.21.15.90:0/2633850754 shutdown_connections 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- 172.21.15.90:0/2633850754 wait complete. 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 Processor -- start 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.020+0000 7fb7cb608700 1 -- start start 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 0x7fb7c40ffc70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41006f0 0x7fb7c41a5370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7c41a5940 con 0x7fb7c41060a0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7c41a5ab0 con 0x7fb7c41006f0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7c41a5c70 con 0x7fb7c40680f0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50484/0 (socket says 172.21.15.90:50484) 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/3937346760 learned_addr learned my addr 172.21.15.90:0/3937346760 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 0x7fb7c40ffc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41006f0 0x7fb7c41a5370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/3937346760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 msgr2=0x7fb7c40ffc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 0x7fb7c40ffc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/3937346760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41006f0 msgr2=0x7fb7c41a5370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41006f0 0x7fb7c41a5370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/3937346760 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7b40095f0 con 0x7fb7c41060a0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7fb7b400b580 tx=0x7fb7b400f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7ba7fc700 1 -- 172.21.15.90:0/3937346760 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7b401c070 con 0x7fb7c41060a0 2024-04-06T14:05:32.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7ba7fc700 1 -- 172.21.15.90:0/3937346760 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb7b400fd80 con 0x7fb7c41060a0 2024-04-06T14:05:32.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7c41a5ef0 con 0x7fb7c41060a0 2024-04-06T14:05:32.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7ba7fc700 1 -- 172.21.15.90:0/3937346760 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7b4022ca0 con 0x7fb7c41060a0 2024-04-06T14:05:32.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.021+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7c41a6310 con 0x7fb7c41060a0 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.023+0000 7fb7ba7fc700 1 -- 172.21.15.90:0/3937346760 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fb7b4020070 con 0x7fb7c41060a0 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.024+0000 7fb7ba7fc700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 0x7fb7b00a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.024+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 0x7fb7b00a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.024+0000 7fb7ba7fc700 1 -- 172.21.15.90:0/3937346760 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb7b40cde10 con 0x7fb7c41060a0 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 0x7fb7b00a38f0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fb7c0009c60 tx=0x7fb7c0009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 Processor -- start 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 -- start start 2024-04-06T14:05:32.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ddb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045e78a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563904535990 con 0x5639045e7de0 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563904533350 con 0x5639045dd780 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.025+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56390452ec40 con 0x5639045de0a0 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50488/0 (socket says 172.21.15.90:50488) 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2359536606 learned_addr learned my addr 172.21.15.90:0/2359536606 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045e78a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ddb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2359536606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 msgr2=0x5639045e78a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045e78a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2359536606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 msgr2=0x5639045ddb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ddb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2359536606 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5639043b03b0 con 0x5639045e7de0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7fb7bc003f30 tx=0x7fb7bc009c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cbe4d6a78a01248d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7af7fe700 1 -- 172.21.15.90:0/2359536606 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7bc003a00 con 0x5639045e7de0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7af7fe700 1 -- 172.21.15.90:0/2359536606 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb7bc00f040 con 0x5639045e7de0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.026+0000 7fb7af7fe700 1 -- 172.21.15.90:0/2359536606 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7bc0156e0 con 0x5639045e7de0 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2359536606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 msgr2=0x5639045ea1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 crypto rx=0x7fb7bc003f30 tx=0x7fb7bc009c20 comp rx=0 tx=0).stop 2024-04-06T14:05:32.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2359536606 shutdown_connections 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ea1d0 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ddb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2359536606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045e78a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2359536606 >> 172.21.15.90:0/2359536606 conn(0x56390453c6a0 msgr2=0x5639045d8a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2359536606 shutdown_connections 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2359536606 wait complete. 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 -- start start 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.027+0000 7fb7d77bcb80 1 Processor -- start 2024-04-06T14:05:32.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 0x5639045e5120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639044cad80 con 0x5639045de0a0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639044caa20 con 0x5639045e7de0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639043b26a0 con 0x5639045dd780 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7d77bcb80 10 client.0 _ll_register_callbacks cb 0x5639044d4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c93a4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50490/0 (socket says 172.21.15.90:50490) 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/254193172 learned_addr learned my addr 172.21.15.90:0/254193172 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c93a4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57040/0 (socket says 172.21.15.90:57040) 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 0x5639045e5120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/254193172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 msgr2=0x5639045e7760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 msgr2=0x5639045e5120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 0x5639045e5120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c8ba3700 1 -- 172.21.15.90:0/254193172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7bc003b60 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 0x5639045e5120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.028+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fb7b4097230 tx=0x7fb7b4022700 comp rx=0 tx=0).ready entity=mon.0 client_cookie=21aba65791bd81a1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7b409b4b0 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb7b409ad30 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5639044ef380 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb7b409eb60 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 10 client.15495 fetch_fsmap learned FSMap version 24 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56390435cb60 con 0x5639045de0a0 2024-04-06T14:05:32.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5639044ef380 con 0x5639045de0a0 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb7b40aeb70 con 0x5639045de0a0 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 10 client.15495 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 10 client.15495 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 20 client.15495 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5639045f7580 0x5639045f99a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 4 client.15495 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.029+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563904527700 con 0x5639045f7580 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.030+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5639045f7580 0x5639045f99a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.030+0000 7fb7acff9700 10 client.15495 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.030+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5639045f7580 0x5639045f99a0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1eb4168307942da9 server_cookie=e34b164ca02152e5 in_seq=0 out_seq=0 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7acff9700 1 -- 172.21.15.90:0/254193172 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563904527700 con 0x5639045f7580 2024-04-06T14:05:32.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7acff9700 10 client.15495 handle_command_reply: tid=0 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 client.15495 shutdown 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 msgr2=0x5639045e4af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 secure :-1 s=READY pgs=451 cs=0 l=1 rev1=1 crypto rx=0x7fb7b4097230 tx=0x7fb7b4022700 comp rx=0 tx=0).stop 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 shutdown_connections 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045e7de0 0x5639045e5120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5639045f7580 0x5639045f99a0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045e7760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/254193172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045de0a0 0x5639045e4af0 unknown :-1 s=CLOSED pgs=451 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7acff9700 0 client.15495 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 >> 172.21.15.90:0/254193172 conn(0x56390453c6a0 msgr2=0x5639045d7c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 shutdown_connections 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/254193172 wait complete. 2024-04-06T14:05:32.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.031+0000 7fb7d77bcb80 20 client.15495 trim_cache size 0 max 16384 2024-04-06T14:05:32.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 Processor -- start 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 -- start start 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045eea60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 0x5639045f38c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045de7a0 con 0x5639045e7de0 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045df250 con 0x5639045de0a0 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045e5a00 con 0x5639045dd780 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50496/0 (socket says 172.21.15.90:50496) 2024-04-06T14:05:32.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/1778953123 learned_addr learned my addr 172.21.15.90:0/1778953123 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 0x5639045f38c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045eea60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/1778953123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 msgr2=0x5639045eea60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045eea60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/1778953123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 msgr2=0x5639045f38c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 0x5639045f38c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/1778953123 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5639044f4ea0 con 0x5639045e7de0 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 0x5639045f38c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045eea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:32.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.048+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x5639045df1f0 tx=0x7fb7bc003a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=738efbdb12379d15 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7affff700 1 -- 172.21.15.90:0/1778953123 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7bc007a10 con 0x5639045e7de0 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7affff700 1 -- 172.21.15.90:0/1778953123 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb7bc007b70 con 0x5639045e7de0 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7affff700 1 -- 172.21.15.90:0/1778953123 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7bc004870 con 0x5639045e7de0 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/1778953123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 msgr2=0x5639045d83a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 secure :-1 s=READY pgs=452 cs=0 l=1 rev1=1 crypto rx=0x5639045df1f0 tx=0x7fb7bc003a00 comp rx=0 tx=0).stop 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/1778953123 shutdown_connections 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045d83a0 unknown :-1 s=CLOSED pgs=452 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045dd780 0x5639045eea60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/1778953123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045de0a0 0x5639045f38c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/1778953123 >> 172.21.15.90:0/1778953123 conn(0x56390453c6a0 msgr2=0x5639045f31b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/1778953123 shutdown_connections 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.049+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/1778953123 wait complete. 2024-04-06T14:05:32.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 -- start start 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 Processor -- start 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ee7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045ece30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045ed9c0 con 0x5639045e7de0 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045edb30 con 0x5639045dd780 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5639045edcf0 con 0x5639045de0a0 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7d77bcb80 10 client.0 _ll_register_callbacks cb 0x5639044d4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50510/0 (socket says 172.21.15.90:50510) 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2449513574 learned_addr learned my addr 172.21.15.90:0/2449513574 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ee7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2449513574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 msgr2=0x5639045ece30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c8ba3700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045ece30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045ece30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 msgr2=0x5639045ee7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.050+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ee7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7c9ba5700 1 -- 172.21.15.90:0/2449513574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5639043b03b0 con 0x5639045e7de0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ee7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7c9ba5700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fb7bc000f80 tx=0x7fb7bc0157e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c4addb565181c317 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb7bc004670 con 0x5639045e7de0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb7bc02f070 con 0x5639045e7de0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb7bc00f040 con 0x5639045e7de0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb7bc015c90 con 0x5639045e7de0 2024-04-06T14:05:32.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 10 client.15501 fetch_fsmap learned FSMap version 24 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5639043b4cd0 con 0x5639045e7de0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb7bc00f040 con 0x5639045e7de0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb7bc03ca10 con 0x5639045e7de0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 10 client.15501 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 10 client.15501 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 20 client.15501 populate_metadata read hostname 'smithi090' 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56390464c4f0 0x56390464e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 4 client.15501 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.051+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56390437dcf0 con 0x56390464c4f0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56390464c4f0 0x56390464e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7acff9700 10 client.15501 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7c93a4700 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56390464c4f0 0x56390464e8e0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3556e5b3e2aa3c0b server_cookie=be017c832f28189a in_seq=0 out_seq=0 2024-04-06T14:05:32.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7acff9700 1 -- 172.21.15.90:0/2449513574 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x56390437dcf0 con 0x56390464c4f0 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7acff9700 10 client.15501 handle_command_reply: tid=0 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 client.15501 shutdown 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 msgr2=0x5639045ed370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 secure :-1 s=READY pgs=453 cs=0 l=1 rev1=1 crypto rx=0x7fb7bc000f80 tx=0x7fb7bc0157e0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 shutdown_connections 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5639045e7de0 0x5639045ed370 unknown :-1 s=CLOSED pgs=453 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5639045dd780 0x5639045ee7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56390464c4f0 0x56390464e8e0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 --2- 172.21.15.90:0/2449513574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5639045de0a0 0x5639045ece30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 >> 172.21.15.90:0/2449513574 conn(0x56390453c6a0 msgr2=0x5639045f31b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.052+0000 7fb7acff9700 0 client.15501 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 shutdown_connections 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7d77bcb80 1 -- 172.21.15.90:0/2449513574 wait complete. 2024-04-06T14:05:32.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7d77bcb80 20 client.15501 trim_cache size 0 max 16384 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "39b3d254-51ce-4d50-a022-eb7f865a85a8": { 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "39b3d254-51ce-4d50-a022-eb7f865a85a8", 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:05:32.062 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:05:32.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 msgr2=0x7fb7b00a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 0x7fb7b00a38f0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fb7c0009c60 tx=0x7fb7c0009260 comp rx=0 tx=0).stop 2024-04-06T14:05:32.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 msgr2=0x7fb7c41001b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 crypto rx=0x7fb7b400b580 tx=0x7fb7b400f690 comp rx=0 tx=0).stop 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 shutdown_connections 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb7b00a1430 0x7fb7b00a38f0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb7c41006f0 0x7fb7c41a5370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb7c40680f0 0x7fb7c40ffc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 --2- 172.21.15.90:0/3937346760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb7c41060a0 0x7fb7c41001b0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 >> 172.21.15.90:0/3937346760 conn(0x7fb7c4076d50 msgr2=0x7fb7c40fe010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 shutdown_connections 2024-04-06T14:05:32.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.053+0000 7fb7cb608700 1 -- 172.21.15.90:0/3937346760 wait complete. 2024-04-06T14:05:32.065 INFO:tasks.cephfs.filesystem:scrub status for tag:39b3d254-51ce-4d50-a022-eb7f865a85a8 - {'path': '/', 'tag': '39b3d254-51ce-4d50-a022-eb7f865a85a8', 'options': 'recursive,force'} 2024-04-06T14:05:32.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.096+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}) v1 -- 0x7ff198112dd0 con 0x7ff1840a1430 2024-04-06T14:05:32.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.160+0000 7ff18effd700 1 -- 172.21.15.90:0/574635654 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7ff198112dd0 con 0x7ff1840a1430 2024-04-06T14:05:32.161 INFO:teuthology.orchestra.run.smithi090.stdout:Schedule set for path / 2024-04-06T14:05:32.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.162+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 msgr2=0x7ff1840a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.162+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 0x7ff1840a38f0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ff194009cb0 tx=0x7ff194009260 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.162+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 msgr2=0x7ff198199a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.162+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 crypto rx=0x7ff18800d980 tx=0x7ff18800dd40 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 shutdown_connections 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1840a1430 0x7ff1840a38f0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff19810d700 0x7ff19819ddd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff198100690 0x7ff1981994f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 --2- 172.21.15.90:0/574635654 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff198100fb0 0x7ff198199a30 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 >> 172.21.15.90:0/574635654 conn(0x7ff198076d20 msgr2=0x7ff19810ee00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 shutdown_connections 2024-04-06T14:05:32.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.163+0000 7ff19fbf4700 1 -- 172.21.15.90:0/574635654 wait complete. 2024-04-06T14:05:32.175 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6m3h' 2024-04-06T14:05:32.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- 172.21.15.90:0/3138200886 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ffbe0 msgr2=0x7f481c10dbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 --2- 172.21.15.90:0/3138200886 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ffbe0 0x7f481c10dbf0 secure :-1 s=READY pgs=454 cs=0 l=1 rev1=1 crypto rx=0x7f481400b0d0 tx=0x7f481400b3e0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- 172.21.15.90:0/3138200886 shutdown_connections 2024-04-06T14:05:32.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 --2- 172.21.15.90:0/3138200886 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ffbe0 0x7f481c10dbf0 unknown :-1 s=CLOSED pgs=454 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 --2- 172.21.15.90:0/3138200886 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c106e30 0x7f481c107210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 --2- 172.21.15.90:0/3138200886 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c0ff220 0x7f481c0ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- 172.21.15.90:0/3138200886 >> 172.21.15.90:0/3138200886 conn(0x7f481c0761b0 msgr2=0x7f481c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- 172.21.15.90:0/3138200886 shutdown_connections 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- 172.21.15.90:0/3138200886 wait complete. 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 Processor -- start 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.340+0000 7f4824781700 1 -- start start 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c0ffbe0 0x7f481c100940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c106e30 0x7f481c100e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481c1a5740 con 0x7f481c0ff220 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481c1a58b0 con 0x7f481c106e30 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481c1a5a70 con 0x7f481c0ffbe0 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50524/0 (socket says 172.21.15.90:50524) 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 -- 172.21.15.90:0/631859 learned_addr learned my addr 172.21.15.90:0/631859 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4822d1e700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c106e30 0x7f481c100e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4821d1c700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c0ffbe0 0x7f481c100940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 -- 172.21.15.90:0/631859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c0ffbe0 msgr2=0x7f481c100940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c0ffbe0 0x7f481c100940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 -- 172.21.15.90:0/631859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c106e30 msgr2=0x7f481c100e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c106e30 0x7f481c100e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 -- 172.21.15.90:0/631859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4814009d00 con 0x7f481c0ff220 2024-04-06T14:05:32.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f482251d700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f481800b530 tx=0x7f481800b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4818011830 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4818011e50 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 -- 172.21.15.90:0/631859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f481c1a5d50 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f481800f2e0 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.341+0000 7f4824781700 1 -- 172.21.15.90:0/631859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f481c1a6200 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.342+0000 7f4824781700 1 -- 172.21.15.90:0/631859 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f481c109340 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.343+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f48180105a0 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.344+0000 7f48137fe700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 0x7f48080a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.344+0000 7f4821d1c700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 0x7f48080a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.344+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f48180ccda0 con 0x7f481c0ff220 2024-04-06T14:05:32.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.345+0000 7f4821d1c700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 0x7f48080a3870 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f481c101ae0 tx=0x7f480c00b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.348+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f48180960c0 con 0x7f481c0ff220 2024-04-06T14:05:32.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.569+0000 7f4824781700 1 -- 172.21.15.90:0/631859 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}) v1 -- 0x7f481c101890 con 0x7f48080a13b0 2024-04-06T14:05:32.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.627+0000 7f48137fe700 1 -- 172.21.15.90:0/631859 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+26 (secure 0 0 0) 0x7f481c101890 con 0x7f48080a13b0 2024-04-06T14:05:32.628 INFO:teuthology.orchestra.run.smithi090.stdout:Retention added to path / 2024-04-06T14:05:32.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 msgr2=0x7f48080a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 0x7f48080a3870 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f481c101ae0 tx=0x7f480c00b410 comp rx=0 tx=0).stop 2024-04-06T14:05:32.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 msgr2=0x7f481c100400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 crypto rx=0x7f481800b530 tx=0x7f481800b840 comp rx=0 tx=0).stop 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 shutdown_connections 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48080a13b0 0x7f48080a3870 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f481c106e30 0x7f481c100e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481c0ff220 0x7f481c100400 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 --2- 172.21.15.90:0/631859 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f481c0ffbe0 0x7f481c100940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 >> 172.21.15.90:0/631859 conn(0x7f481c0761b0 msgr2=0x7f481c103c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 shutdown_connections 2024-04-06T14:05:32.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.629+0000 7f4824781700 1 -- 172.21.15.90:0/631859 wait complete. 2024-04-06T14:05:32.642 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-04-06T14:05:32.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 -- 172.21.15.90:0/2162245346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 msgr2=0x7fbc5c069420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 --2- 172.21.15.90:0/2162245346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c069420 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 crypto rx=0x7fbc58009990 tx=0x7fbc58009ca0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 -- 172.21.15.90:0/2162245346 shutdown_connections 2024-04-06T14:05:32.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 --2- 172.21.15.90:0/2162245346 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c10dba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 --2- 172.21.15.90:0/2162245346 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c107200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 --2- 172.21.15.90:0/2162245346 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c069420 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 -- 172.21.15.90:0/2162245346 >> 172.21.15.90:0/2162245346 conn(0x7fbc5c076a20 msgr2=0x7fbc5c076e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 -- 172.21.15.90:0/2162245346 shutdown_connections 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.764+0000 7fbc63490700 1 -- 172.21.15.90:0/2162245346 wait complete. 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 Processor -- start 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 -- start start 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c1976f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c19ba90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc5c19c0d0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc5c19c240 con 0x7fbc5c106e20 2024-04-06T14:05:32.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc63490700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc5c19c400 con 0x7fbc5c069960 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50540/0 (socket says 172.21.15.90:50540) 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 -- 172.21.15.90:0/37328767 learned_addr learned my addr 172.21.15.90:0/37328767 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc61a2d700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c19ba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc60a2b700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c1976f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 -- 172.21.15.90:0/37328767 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 msgr2=0x7fbc5c1976f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c1976f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 -- 172.21.15.90:0/37328767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 msgr2=0x7fbc5c19ba90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c19ba90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 -- 172.21.15.90:0/37328767 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc580095f0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc60a2b700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c1976f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc61a2d700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c19ba90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.765+0000 7fbc6122c700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fbc4c00d950 tx=0x7fbc4c00dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.766+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc4c009b40 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.766+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc5c19c6e0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.766+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc4c009ca0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.766+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc4c00f620 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.766+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc5c1a3f20 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.767+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc5c1092f0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.768+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fbc4c010460 con 0x7fbc5c068fa0 2024-04-06T14:05:32.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.768+0000 7fbc527fc700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 0x7fbc480a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:32.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.768+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbc4c0cd2d0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.768+0000 7fbc60a2b700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 0x7fbc480a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:32.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.769+0000 7fbc60a2b700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 0x7fbc480a3870 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fbc5800b2e0 tx=0x7fbc58005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:32.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.772+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbc4c0964e0 con 0x7fbc5c068fa0 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 1.3 KiB/s wr, 23 op/s 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/962828173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:05:32.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:32 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:32.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:32.994+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) v1 -- 0x7fbc5c1984b0 con 0x7fbc480a13b0 2024-04-06T14:05:32.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 1.3 KiB/s wr, 23 op/s 2024-04-06T14:05:32.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/962828173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:32.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:05:32.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:32.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:05:32.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:32 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:33.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.002+0000 7fbc527fc700 1 -- 172.21.15.90:0/37328767 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+298 (secure 0 0 0) 0x7fbc5c1984b0 con 0x7fbc480a13b0 2024-04-06T14:05:33.002 INFO:teuthology.orchestra.run.smithi090.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-04-06T00:00:00", "created": "2024-04-06T14:05:32", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 msgr2=0x7fbc480a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 0x7fbc480a3870 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fbc5800b2e0 tx=0x7fbc58005fb0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 msgr2=0x7fbc5c1971b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 crypto rx=0x7fbc4c00d950 tx=0x7fbc4c00dd10 comp rx=0 tx=0).stop 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 shutdown_connections 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc480a13b0 0x7fbc480a3870 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc5c106e20 0x7fbc5c19ba90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc5c068fa0 0x7fbc5c1971b0 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 --2- 172.21.15.90:0/37328767 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc5c069960 0x7fbc5c1976f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 >> 172.21.15.90:0/37328767 conn(0x7fbc5c076a20 msgr2=0x7fbc5c0fedc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:33.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 shutdown_connections 2024-04-06T14:05:33.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.004+0000 7fbc63490700 1 -- 172.21.15.90:0/37328767 wait complete. 2024-04-06T14:05:33.015 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: pgmap v7: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 1.3 KiB/s wr, 23 op/s 2024-04-06T14:05:33.016 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/962828173' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:05:33.016 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:05:33.016 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:33.016 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:05:33.016 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:32 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:05:33.017 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-04-06T14:05:33.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 -- 172.21.15.90:0/3740096306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd8100570 msgr2=0x7fbcd810ca10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/3740096306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd8100570 0x7fbcd810ca10 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 crypto rx=0x7fbcd000b0a0 tx=0x7fbcd000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 -- 172.21.15.90:0/3740096306 shutdown_connections 2024-04-06T14:05:33.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/3740096306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd8100570 0x7fbcd810ca10 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/3740096306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8105c70 0x7fbcd8106050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/3740096306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8100030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 -- 172.21.15.90:0/3740096306 >> 172.21.15.90:0/3740096306 conn(0x7fbcd8076de0 msgr2=0x7fbcd80771f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 -- 172.21.15.90:0/3740096306 shutdown_connections 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.140+0000 7fbcdd67b700 1 -- 172.21.15.90:0/3740096306 wait complete. 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 Processor -- start 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 -- start start 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 0x7fbcd8101da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd8105c70 0x7fbcd8102300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd81a5610 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd81a5780 con 0x7fbcd8105c70 2024-04-06T14:05:33.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcdd67b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbcd81a5940 con 0x7fbcd8100570 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcd6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcd6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:50564/0 (socket says 172.21.15.90:50564) 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcd6ffd700 1 -- 172.21.15.90:0/51378634 learned_addr learned my addr 172.21.15.90:0/51378634 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcd77fe700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd8105c70 0x7fbcd8102300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.141+0000 7fbcd67fc700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 0x7fbcd8101da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 -- 172.21.15.90:0/51378634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 msgr2=0x7fbcd8101da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 0x7fbcd8101da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 -- 172.21.15.90:0/51378634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd8105c70 msgr2=0x7fbcd8102300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd8105c70 0x7fbcd8102300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 -- 172.21.15.90:0/51378634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbcd0009d00 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd67fc700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 0x7fbcd8101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcd6ffd700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fbcc800c920 tx=0x7fbcc800cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcc8007b00 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbcc800ce70 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbcd81a5c20 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbcc80054e0 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.142+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbcd81a6130 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.143+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbcd81012a0 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.144+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fbcc8004750 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.144+0000 7fbccffff700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 0x7fbcc40a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:05:33.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.145+0000 7fbcd67fc700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 0x7fbcc40a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:05:33.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.145+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbcc80cce70 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.145+0000 7fbcd67fc700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 0x7fbcc40a3870 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbcc0005950 tx=0x7fbcc00058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:05:33.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.148+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbcc8096080 con 0x7fbcd80ffbb0 2024-04-06T14:05:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.369+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) v1 -- 0x7fbcd810fb70 con 0x7fbcc40a13b0 2024-04-06T14:05:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.376+0000 7fbccffff700 1 -- 172.21.15.90:0/51378634 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+10 (secure 0 0 0) 0x7fbcd810fb70 con 0x7fbcc40a13b0 2024-04-06T14:05:33.377 INFO:teuthology.orchestra.run.smithi090.stdout:/ 1m 6m3h 2024-04-06T14:05:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.378+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 msgr2=0x7fbcc40a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.378+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 0x7fbcc40a3870 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbcc0005950 tx=0x7fbcc00058e0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.378+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 msgr2=0x7fbcd8101860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:05:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.378+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 crypto rx=0x7fbcc800c920 tx=0x7fbcc800cc30 comp rx=0 tx=0).stop 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 shutdown_connections 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbcc40a13b0 0x7fbcc40a3870 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbcd8105c70 0x7fbcd8102300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbcd80ffbb0 0x7fbcd8101860 unknown :-1 s=CLOSED pgs=459 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 --2- 172.21.15.90:0/51378634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbcd8100570 0x7fbcd8101da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 >> 172.21.15.90:0/51378634 conn(0x7fbcd8076de0 msgr2=0x7fbcd80697a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 shutdown_connections 2024-04-06T14:05:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:05:33.379+0000 7fbcdd67b700 1 -- 172.21.15.90:0/51378634 wait complete. 2024-04-06T14:05:33.391 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > START_TIME' 2024-04-06T14:05:33.419 INFO:teuthology.run_tasks:Running task full_sequential_finally... 2024-04-06T14:05:33.431 INFO:teuthology.run_tasks:Running task exec... 2024-04-06T14:05:33.440 INFO:teuthology.task.exec:Executing custom commands... 2024-04-06T14:05:33.440 INFO:teuthology.task.exec:Running commands on role client.0 host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:05:33.440 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -l /tmp/cephfs/ && sudo df -h /tmp/cephfs/' 2024-04-06T14:05:33.497 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:33.512 INFO:teuthology.orchestra.run.smithi090.stdout:Filesystem Size Used Avail Use% Mounted on 2024-04-06T14:05:33.512 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:3300,172.21.15.105:3300,172.21.15.134:3300:/volumes/qa/sv_1/b6ecb66e-1758-4685-8e5c-896930794149 340G 0 340G 0% /tmp/cephfs 2024-04-06T14:05:33.514 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo mkdir -m 755 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres && sudo chown postgres:postgres /tmp/cephfs/postgres' 2024-04-06T14:05:33.609 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- mkdir -m 700 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres/data' 2024-04-06T14:05:33.643 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:33 smithi090 ceph-mon[43420]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:33.643 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:33 smithi090 ceph-mon[43420]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:33.651 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo semanage fcontext -a -t postgresql_db_t "/tmp/cephfs/postgres(/.*)?"' 2024-04-06T14:05:33.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:33 smithi134 ceph-mon[47549]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:33.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:33 smithi134 ceph-mon[47549]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:33.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:33 smithi105 ceph-mon[46969]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1m", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:33.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:33 smithi105 ceph-mon[46969]: from='client.15507 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6m3h", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:34.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:34 smithi134 ceph-mon[47549]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:34.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:34 smithi134 ceph-mon[47549]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:34.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:34 smithi134 ceph-mon[47549]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 1.0 KiB/s wr, 18 op/s 2024-04-06T14:05:34.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:34 smithi105 ceph-mon[46969]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:34.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:34 smithi105 ceph-mon[46969]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:34.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:34 smithi105 ceph-mon[46969]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 1.0 KiB/s wr, 18 op/s 2024-04-06T14:05:35.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:34 smithi090 ceph-mon[43420]: from='client.15513 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:35.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:34 smithi090 ceph-mon[43420]: from='client.15519 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:05:35.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:34 smithi090 ceph-mon[43420]: pgmap v8: 97 pgs: 97 active+clean; 1.1 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 1.0 KiB/s wr, 18 op/s 2024-04-06T14:05:35.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:35 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:05:35.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:35 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:05:36.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:35 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:05:36.536 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo grep -i postgresql /etc/selinux/targeted/contexts/files/file_contexts.local' 2024-04-06T14:05:36.576 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres(/.*)? system_u:object_r:postgresql_db_t:s0 2024-04-06T14:05:36.578 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo restorecon -R -v /tmp/cephfs/postgres' 2024-04-06T14:05:36.660 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -lZaR /tmp/cephfs/postgres/' 2024-04-06T14:05:36.724 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:36 smithi090 ceph-mon[43420]: pgmap v9: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 85 KiB/s wr, 53 op/s 2024-04-06T14:05:36.738 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/: 2024-04-06T14:05:36.738 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:36.738 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:36.738 INFO:teuthology.orchestra.run.smithi090.stdout:drwxrwxrwt. 3 root root system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:05:36.738 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 data 2024-04-06T14:05:36.739 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:36.739 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data: 2024-04-06T14:05:36.739 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:36.739 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:36.739 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:05:36.740 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo mkdir -p /etc/systemd/system/postgresql.service.d/ && printf '"'"'[Service]\nEnvironment=PGDATA=/tmp/cephfs/postgres/data\nEnvironment=PGLOG=/tmp/cephfs/postgres/pgstartup.log\n'"'"' | sudo tee /etc/systemd/system/postgresql.service.d/env.conf' 2024-04-06T14:05:36.827 INFO:teuthology.orchestra.run.smithi090.stdout:[Service] 2024-04-06T14:05:36.827 INFO:teuthology.orchestra.run.smithi090.stdout:Environment=PGDATA=/tmp/cephfs/postgres/data 2024-04-06T14:05:36.827 INFO:teuthology.orchestra.run.smithi090.stdout:Environment=PGLOG=/tmp/cephfs/postgres/pgstartup.log 2024-04-06T14:05:36.829 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- postgresql-setup --initdb' 2024-04-06T14:05:36.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:36 smithi134 ceph-mon[47549]: pgmap v9: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 85 KiB/s wr, 53 op/s 2024-04-06T14:05:36.906 INFO:teuthology.orchestra.run.smithi090.stderr: * Initializing database in '/tmp/cephfs/postgres/data' 2024-04-06T14:05:36.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:36 smithi105 ceph-mon[46969]: pgmap v9: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 85 KiB/s wr, 53 op/s 2024-04-06T14:05:38.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:38 smithi090 ceph-mon[43420]: pgmap v10: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:38 smithi134 ceph-mon[47549]: pgmap v10: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:38 smithi105 ceph-mon[46969]: pgmap v10: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:39.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:39 smithi105 ceph-mon[46969]: pgmap v11: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:40.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:39 smithi090 ceph-mon[43420]: pgmap v11: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:39 smithi134 ceph-mon[47549]: pgmap v11: 97 pgs: 97 active+clean; 1.3 MiB data, 352 MiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 78 KiB/s wr, 48 op/s 2024-04-06T14:05:41.067 INFO:teuthology.orchestra.run.smithi090.stderr: * Initialized, logs are in /var/lib/pgsql/initdb_postgresql.log 2024-04-06T14:05:41.070 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -lZaR /tmp/cephfs/postgres/' 2024-04-06T14:05:41.115 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/: 2024-04-06T14:05:41.115 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:drwxrwxrwt. 3 root root system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 data 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data: 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:total 30 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 . 2024-04-06T14:05:41.116 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 base 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 58 Apr 6 14:05 global 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 log 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_commit_ts 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_dynshmem 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 4269 Apr 6 14:05 pg_hba.conf 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 1636 Apr 6 14:05 pg_ident.conf 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 pg_logical 2024-04-06T14:05:41.117 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 pg_multixact 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 pg_notify 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_replslot 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_serial 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_snapshots 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_stat 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_stat_tmp 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 pg_subtrans 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_tblspc 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_twophase 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 pg_wal 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 pg_xact 2024-04-06T14:05:41.118 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 88 Apr 6 14:05 postgresql.auto.conf 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 22989 Apr 6 14:05 postgresql.conf 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base: 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 . 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 1 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 12346 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 12347 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/1: 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:total 7233 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 . 2024-04-06T14:05:41.119 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 .. 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:05:41.120 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:05:41.121 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:05:41.122 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:05 1259 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:05:41.123 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:05:41.124 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:05:41.125 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Apr 6 14:05 2608 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:05:41.126 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:05:41.127 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Apr 6 14:05 2619 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2619_fsm 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:05:41.128 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Apr 6 14:05 2658 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:05:41.129 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:05:41.130 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2696 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:05:41.131 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:05:41.132 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:05:41.133 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:05:41.134 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:05:41.135 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:05:41.136 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:05:41.137 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:05:41.138 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:05:41.139 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:05:41.139 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:05:41.139 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:05:41.143 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:05:41.144 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/12346: 2024-04-06T14:05:41.145 INFO:teuthology.orchestra.run.smithi090.stdout:total 7233 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 . 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 .. 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:05:41.146 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:05:41.147 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:05:41.148 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:05:41.149 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:05 1259 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:05:41.150 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:05:41.151 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:05:41.152 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:05:41.153 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Apr 6 14:05 2608 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:05:41.154 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:05:41.155 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Apr 6 14:05 2619 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2619_fsm 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Apr 6 14:05 2658 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:05:41.156 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:05:41.157 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:05:41.158 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2696 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:05:41.159 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:05:41.160 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:05:41.161 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:05:41.162 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:05:41.163 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:05:41.164 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:05:41.165 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:05:41.166 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:05:41.167 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/12347: 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:total 7233 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 . 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 .. 2024-04-06T14:05:41.168 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:05:41.169 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:05:41.170 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:05:41.171 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:05 1259 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:05:41.172 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:05:41.173 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:05:41.174 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Apr 6 14:05 2608 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:05:41.175 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:05:41.176 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Apr 6 14:05 2619 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2619_fsm 2024-04-06T14:05:41.177 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:05:41.178 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Apr 6 14:05 2658 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:05:41.179 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:05:41.180 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2696 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:05:41.181 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:05:41.182 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:05:41.183 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:05:41.184 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:05:41.185 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:05:41.186 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:05:41.187 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:05:41.188 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:05:41.189 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:05:41.190 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:05:41.191 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:05:41.192 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:05:41.193 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/global: 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:total 553 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 58 Apr 6 14:05 . 2024-04-06T14:05:41.194 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1136 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1136_fsm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1136_vm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1137 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1213 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1213_fsm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1213_vm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1214 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1214_fsm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1214_vm 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1232 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1233 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1260 2024-04-06T14:05:41.195 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1260_fsm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1260_vm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1261 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1261_fsm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1261_vm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1262 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1262_fsm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1262_vm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2396 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2396_fsm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2396_vm 2024-04-06T14:05:41.196 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2397 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2671 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2672 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2676 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2677 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2694 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2695 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2697 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2698 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2846 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2846_vm 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2847 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2964 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2964_vm 2024-04-06T14:05:41.197 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2965 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2966 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2966_vm 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2967 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3592 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3592_vm 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3593 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 4060 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 4060_vm 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 4061 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6000 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6000_vm 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6001 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6002 2024-04-06T14:05:41.198 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6100 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6100_vm 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6114 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6115 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 pg_control 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/log: 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_commit_ts: 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.199 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_dynshmem: 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical: 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:total 1 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 . 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 mappings 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8 Apr 6 14:05 replorigin_checkpoint 2024-04-06T14:05:41.200 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 snapshots 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical/mappings: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 .. 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical/snapshots: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 .. 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact: 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 . 2024-04-06T14:05:41.201 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 members 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 offsets 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact/members: 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 .. 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact/offsets: 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 .. 2024-04-06T14:05:41.202 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_notify: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_replslot: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_serial: 2024-04-06T14:05:41.203 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_snapshots: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_stat: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_stat_tmp: 2024-04-06T14:05:41.204 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_subtrans: 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_tblspc: 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.205 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_twophase: 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_wal: 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:total 16384 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 . 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:05 000000010000000000000001 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 archive_status 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_wal/archive_status: 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:05:41.206 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 .. 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_xact: 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Apr 6 14:05 .. 2024-04-06T14:05:41.207 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:05:41.207 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo systemctl start postgresql' 2024-04-06T14:05:41.317 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- pgbench -s 32 -i' 2024-04-06T14:05:41.363 INFO:teuthology.orchestra.run.smithi090.stderr:NOTICE: table "pgbench_history" does not exist, skipping 2024-04-06T14:05:41.367 INFO:teuthology.orchestra.run.smithi090.stderr:NOTICE: table "pgbench_tellers" does not exist, skipping 2024-04-06T14:05:41.370 INFO:teuthology.orchestra.run.smithi090.stderr:NOTICE: table "pgbench_accounts" does not exist, skipping 2024-04-06T14:05:41.373 INFO:teuthology.orchestra.run.smithi090.stderr:NOTICE: table "pgbench_branches" does not exist, skipping 2024-04-06T14:05:41.392 INFO:teuthology.orchestra.run.smithi090.stderr:creating tables... 2024-04-06T14:05:41.451 INFO:teuthology.orchestra.run.smithi090.stderr:100000 of 3200000 tuples (3%) done (elapsed 0.06 s, remaining 1.77 s) 2024-04-06T14:05:41.631 INFO:teuthology.orchestra.run.smithi090.stderr:200000 of 3200000 tuples (6%) done (elapsed 0.24 s, remaining 3.55 s) 2024-04-06T14:05:41.725 INFO:teuthology.orchestra.run.smithi090.stderr:300000 of 3200000 tuples (9%) done (elapsed 0.33 s, remaining 3.20 s) 2024-04-06T14:05:41.880 INFO:teuthology.orchestra.run.smithi090.stderr:400000 of 3200000 tuples (12%) done (elapsed 0.49 s, remaining 3.40 s) 2024-04-06T14:05:41.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:41 smithi105 ceph-mon[46969]: pgmap v12: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 2.6 MiB/s wr, 91 op/s 2024-04-06T14:05:42.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:41 smithi090 ceph-mon[43420]: pgmap v12: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 2.6 MiB/s wr, 91 op/s 2024-04-06T14:05:42.060 INFO:teuthology.orchestra.run.smithi090.stderr:500000 of 3200000 tuples (15%) done (elapsed 0.67 s, remaining 3.60 s) 2024-04-06T14:05:42.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:41 smithi134 ceph-mon[47549]: pgmap v12: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 2.6 MiB/s wr, 91 op/s 2024-04-06T14:05:42.218 INFO:teuthology.orchestra.run.smithi090.stderr:600000 of 3200000 tuples (18%) done (elapsed 0.82 s, remaining 3.57 s) 2024-04-06T14:05:42.291 INFO:teuthology.orchestra.run.smithi090.stderr:700000 of 3200000 tuples (21%) done (elapsed 0.90 s, remaining 3.20 s) 2024-04-06T14:05:42.472 INFO:teuthology.orchestra.run.smithi090.stderr:800000 of 3200000 tuples (25%) done (elapsed 1.08 s, remaining 3.24 s) 2024-04-06T14:05:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:900000 of 3200000 tuples (28%) done (elapsed 1.24 s, remaining 3.16 s) 2024-04-06T14:05:42.710 INFO:teuthology.orchestra.run.smithi090.stderr:1000000 of 3200000 tuples (31%) done (elapsed 1.32 s, remaining 2.89 s) 2024-04-06T14:05:42.870 INFO:teuthology.orchestra.run.smithi090.stderr:1100000 of 3200000 tuples (34%) done (elapsed 1.48 s, remaining 2.82 s) 2024-04-06T14:05:42.951 INFO:teuthology.orchestra.run.smithi090.stderr:1200000 of 3200000 tuples (37%) done (elapsed 1.56 s, remaining 2.60 s) 2024-04-06T14:05:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:1300000 of 3200000 tuples (40%) done (elapsed 1.74 s, remaining 2.54 s) 2024-04-06T14:05:43.293 INFO:teuthology.orchestra.run.smithi090.stderr:1400000 of 3200000 tuples (43%) done (elapsed 1.90 s, remaining 2.44 s) 2024-04-06T14:05:43.373 INFO:teuthology.orchestra.run.smithi090.stderr:1500000 of 3200000 tuples (46%) done (elapsed 1.98 s, remaining 2.24 s) 2024-04-06T14:05:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:1600000 of 3200000 tuples (50%) done (elapsed 2.17 s, remaining 2.17 s) 2024-04-06T14:05:43.749 INFO:teuthology.orchestra.run.smithi090.stderr:1700000 of 3200000 tuples (53%) done (elapsed 2.35 s, remaining 2.08 s) 2024-04-06T14:05:43.811 INFO:teuthology.orchestra.run.smithi090.stderr:1800000 of 3200000 tuples (56%) done (elapsed 2.42 s, remaining 1.88 s) 2024-04-06T14:05:43.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:43 smithi105 ceph-mon[46969]: pgmap v13: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 2.6 MiB/s wr, 77 op/s 2024-04-06T14:05:44.004 INFO:teuthology.orchestra.run.smithi090.stderr:1900000 of 3200000 tuples (59%) done (elapsed 2.61 s, remaining 1.79 s) 2024-04-06T14:05:44.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:43 smithi090 ceph-mon[43420]: pgmap v13: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 2.6 MiB/s wr, 77 op/s 2024-04-06T14:05:44.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:43 smithi134 ceph-mon[47549]: pgmap v13: 97 pgs: 97 active+clean; 31 MiB data, 373 MiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 2.6 MiB/s wr, 77 op/s 2024-04-06T14:05:44.324 INFO:teuthology.orchestra.run.smithi090.stderr:2000000 of 3200000 tuples (62%) done (elapsed 2.79 s, remaining 1.67 s) 2024-04-06T14:05:44.324 INFO:teuthology.orchestra.run.smithi090.stderr:2100000 of 3200000 tuples (65%) done (elapsed 2.85 s, remaining 1.49 s) 2024-04-06T14:05:44.472 INFO:teuthology.orchestra.run.smithi090.stderr:2200000 of 3200000 tuples (68%) done (elapsed 3.08 s, remaining 1.40 s) 2024-04-06T14:05:44.548 INFO:teuthology.orchestra.run.smithi090.stderr:2300000 of 3200000 tuples (71%) done (elapsed 3.15 s, remaining 1.23 s) 2024-04-06T14:05:44.723 INFO:teuthology.orchestra.run.smithi090.stderr:2400000 of 3200000 tuples (75%) done (elapsed 3.33 s, remaining 1.11 s) 2024-04-06T14:05:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2500000 of 3200000 tuples (78%) done (elapsed 3.52 s, remaining 0.99 s) 2024-04-06T14:05:44.981 INFO:teuthology.orchestra.run.smithi090.stderr:2600000 of 3200000 tuples (81%) done (elapsed 3.59 s, remaining 0.83 s) 2024-04-06T14:05:45.153 INFO:teuthology.orchestra.run.smithi090.stderr:2700000 of 3200000 tuples (84%) done (elapsed 3.76 s, remaining 0.70 s) 2024-04-06T14:05:45.247 INFO:teuthology.orchestra.run.smithi090.stderr:2800000 of 3200000 tuples (87%) done (elapsed 3.85 s, remaining 0.55 s) 2024-04-06T14:05:45.408 INFO:teuthology.orchestra.run.smithi090.stderr:2900000 of 3200000 tuples (90%) done (elapsed 4.01 s, remaining 0.42 s) 2024-04-06T14:05:45.581 INFO:teuthology.orchestra.run.smithi090.stderr:3000000 of 3200000 tuples (93%) done (elapsed 4.19 s, remaining 0.28 s) 2024-04-06T14:05:45.737 INFO:teuthology.orchestra.run.smithi090.stderr:3100000 of 3200000 tuples (96%) done (elapsed 4.34 s, remaining 0.14 s) 2024-04-06T14:05:45.823 INFO:teuthology.orchestra.run.smithi090.stderr:3200000 of 3200000 tuples (100%) done (elapsed 4.43 s, remaining 0.00 s) 2024-04-06T14:05:45.853 INFO:teuthology.orchestra.run.smithi090.stderr:vacuum... 2024-04-06T14:05:45.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:45 smithi105 ceph-mon[46969]: pgmap v14: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 368 KiB/s rd, 41 MiB/s wr, 165 op/s 2024-04-06T14:05:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:45 smithi090 ceph-mon[43420]: pgmap v14: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 368 KiB/s rd, 41 MiB/s wr, 165 op/s 2024-04-06T14:05:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:45 smithi134 ceph-mon[47549]: pgmap v14: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 368 KiB/s rd, 41 MiB/s wr, 165 op/s 2024-04-06T14:05:46.447 INFO:teuthology.orchestra.run.smithi090.stderr:set primary keys... 2024-04-06T14:05:47.963 INFO:teuthology.orchestra.run.smithi090.stderr:done. 2024-04-06T14:05:47.966 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- pgbench -c 100 -j 4 --progress=5 --time=900' 2024-04-06T14:05:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:starting vacuum...end. 2024-04-06T14:05:48.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:48 smithi090 ceph-mon[43420]: pgmap v15: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:48.896 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:48 smithi134 ceph-mon[47549]: pgmap v15: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:48 smithi105 ceph-mon[46969]: pgmap v15: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:49 smithi105 ceph-mon[46969]: pgmap v16: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:50.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:49 smithi090 ceph-mon[43420]: pgmap v16: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:50.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:49 smithi134 ceph-mon[47549]: pgmap v16: 97 pgs: 97 active+clean; 267 MiB data, 478 MiB used, 1.0 TiB / 1.0 TiB avail; 347 KiB/s rd, 41 MiB/s wr, 131 op/s 2024-04-06T14:05:51.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:51 smithi105 ceph-mon[46969]: pgmap v17: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 391 KiB/s rd, 85 MiB/s wr, 278 op/s 2024-04-06T14:05:52.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:51 smithi090 ceph-mon[43420]: pgmap v17: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 391 KiB/s rd, 85 MiB/s wr, 278 op/s 2024-04-06T14:05:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:51 smithi134 ceph-mon[47549]: pgmap v17: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 391 KiB/s rd, 85 MiB/s wr, 278 op/s 2024-04-06T14:05:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 5.0 s, 3809.1 tps, lat 25.848 ms stddev 31.107 2024-04-06T14:05:53.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:53 smithi105 ceph-mon[46969]: pgmap v18: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 313 KiB/s rd, 83 MiB/s wr, 235 op/s 2024-04-06T14:05:54.044 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:53 smithi090 ceph-mon[43420]: pgmap v18: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 313 KiB/s rd, 83 MiB/s wr, 235 op/s 2024-04-06T14:05:54.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:53 smithi134 ceph-mon[47549]: pgmap v18: 97 pgs: 97 active+clean; 893 MiB data, 731 MiB used, 1.0 TiB / 1.0 TiB avail; 313 KiB/s rd, 83 MiB/s wr, 235 op/s 2024-04-06T14:05:55.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:55 smithi105 ceph-mon[46969]: pgmap v19: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 344 KiB/s rd, 146 MiB/s wr, 882 op/s 2024-04-06T14:05:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:55 smithi090 ceph-mon[43420]: pgmap v19: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 344 KiB/s rd, 146 MiB/s wr, 882 op/s 2024-04-06T14:05:56.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:55 smithi134 ceph-mon[47549]: pgmap v19: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 344 KiB/s rd, 146 MiB/s wr, 882 op/s 2024-04-06T14:05:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 10.0 s, 4385.7 tps, lat 22.829 ms stddev 39.914 2024-04-06T14:05:58.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:58 smithi090 ceph-mon[43420]: pgmap v20: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 107 MiB/s wr, 794 op/s 2024-04-06T14:05:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:58 smithi134 ceph-mon[47549]: pgmap v20: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 107 MiB/s wr, 794 op/s 2024-04-06T14:05:58.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:58 smithi105 ceph-mon[46969]: pgmap v20: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 107 MiB/s wr, 794 op/s 2024-04-06T14:05:59.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:05:59 smithi105 ceph-mon[46969]: pgmap v21: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 109 MiB/s wr, 819 op/s 2024-04-06T14:06:00.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:05:59 smithi090 ceph-mon[43420]: pgmap v21: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 109 MiB/s wr, 819 op/s 2024-04-06T14:06:00.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:05:59 smithi134 ceph-mon[47549]: pgmap v21: 97 pgs: 97 active+clean; 1.1 GiB data, 1.0 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 109 MiB/s wr, 819 op/s 2024-04-06T14:06:01.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:06:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:06:00.646+0000 7f916a0e5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:06:01.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:06:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:06:00.646+0000 7f916a0e5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:06:01.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:06:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:06:00.646+0000 7f916a0e5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:06:01.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:06:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:06:00.646+0000 7f916a0e5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:06:01.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:06:00 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:06:00.646+0000 7f916a0e5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-06T14:06:01.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:01 smithi105 ceph-mon[46969]: pgmap v22: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 129 MiB/s wr, 1.17k op/s 2024-04-06T14:06:02.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:01 smithi090 ceph-mon[43420]: pgmap v22: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 129 MiB/s wr, 1.17k op/s 2024-04-06T14:06:02.067 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:06:02.067 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:06:02.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:01 smithi134 ceph-mon[47549]: pgmap v22: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 129 MiB/s wr, 1.17k op/s 2024-04-06T14:06:02.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 -- 172.21.15.90:0/3049962043 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 msgr2=0x7f4810100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 --2- 172.21.15.90:0/3049962043 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f4810100250 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f48000098f0 tx=0x7f4800009c00 comp rx=0 tx=0).stop 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 -- 172.21.15.90:0/3049962043 shutdown_connections 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 --2- 172.21.15.90:0/3049962043 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 0x7f481010caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 --2- 172.21.15.90:0/3049962043 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f4810100250 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 --2- 172.21.15.90:0/3049962043 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4810100820 0x7f4810104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 -- 172.21.15.90:0/3049962043 >> 172.21.15.90:0/3049962043 conn(0x7f4810076d50 msgr2=0x7f4810077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 -- 172.21.15.90:0/3049962043 shutdown_connections 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.258+0000 7f4815b3c700 1 -- 172.21.15.90:0/3049962043 wait complete. 2024-04-06T14:06:02.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.259+0000 7f4815b3c700 1 Processor -- start 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.259+0000 7f4815b3c700 1 -- start start 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.259+0000 7f4815b3c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.259+0000 7f4815b3c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 0x7f48101966f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.259+0000 7f4815b3c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481019aa90 0x7f481019af10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f4815b3c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481019b510 con 0x7f481019aa90 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f4815b3c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481019b680 con 0x7f4810104f70 2024-04-06T14:06:02.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f4815b3c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f481019b840 con 0x7f48100ffe70 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45280/0 (socket says 172.21.15.90:45280) 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 -- 172.21.15.90:0/2349243250 learned_addr learned my addr 172.21.15.90:0/2349243250 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480effd700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 0x7f48101966f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480ffff700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481019aa90 0x7f481019af10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 -- 172.21.15.90:0/2349243250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 msgr2=0x7f48101966f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 0x7f48101966f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 -- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481019aa90 msgr2=0x7f481019af10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481019aa90 0x7f481019af10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 -- 172.21.15.90:0/2349243250 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4800009520 con 0x7f48100ffe70 2024-04-06T14:06:02.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480f7fe700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f4800009810 tx=0x7f480000fab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480cff9700 1 -- 172.21.15.90:0/2349243250 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f480001c070 con 0x7f48100ffe70 2024-04-06T14:06:02.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.260+0000 7f480cff9700 1 -- 172.21.15.90:0/2349243250 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4800005530 con 0x7f48100ffe70 2024-04-06T14:06:02.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.261+0000 7f480cff9700 1 -- 172.21.15.90:0/2349243250 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4800003dd0 con 0x7f48100ffe70 2024-04-06T14:06:02.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.261+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4810067e00 con 0x7f48100ffe70 2024-04-06T14:06:02.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.261+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4810068170 con 0x7f48100ffe70 2024-04-06T14:06:02.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.263+0000 7f480cff9700 1 -- 172.21.15.90:0/2349243250 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7f480002a440 con 0x7f48100ffe70 2024-04-06T14:06:02.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.264+0000 7f480cff9700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 0x7f47f80a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.264+0000 7f480cff9700 1 -- 172.21.15.90:0/2349243250 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f48000ce010 con 0x7f48100ffe70 2024-04-06T14:06:02.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.264+0000 7f480effd700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 0x7f47f80a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f480effd700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 0x7f47f80a38f0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4804005950 tx=0x7f48040058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 Processor -- start 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 -- start start 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a0eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a210b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 0x55c728a239e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728966990 con 0x55c728a215f0 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728964350 con 0x55c728a0f0a0 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c72895fc40 con 0x55c728a0e780 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f480ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 0x55c728a239e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.265+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a0eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a0eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45296/0 (socket says 172.21.15.90:45296) 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480f7fe700 1 -- 172.21.15.90:0/1031824033 learned_addr learned my addr 172.21.15.90:0/1031824033 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480effd700 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a210b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 -- 172.21.15.90:0/1031824033 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 msgr2=0x55c728a0eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a0eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 -- 172.21.15.90:0/1031824033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 msgr2=0x55c728a210b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a210b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 -- 172.21.15.90:0/1031824033 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c7287e13b0 con 0x55c728a215f0 2024-04-06T14:06:02.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480effd700 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a210b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:02.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.266+0000 7f480ffff700 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 0x55c728a239e0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f4808009e60 tx=0x7f4808009e90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d59756742f0006c2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.267+0000 7f47f5ffb700 1 -- 172.21.15.90:0/1031824033 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808011070 con 0x55c728a215f0 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.267+0000 7f47f5ffb700 1 -- 172.21.15.90:0/1031824033 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f480800ce90 con 0x55c728a215f0 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.267+0000 7f47f5ffb700 1 -- 172.21.15.90:0/1031824033 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808015d70 con 0x55c728a215f0 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.267+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1031824033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 msgr2=0x55c728a239e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.267+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 0x55c728a239e0 secure :-1 s=READY pgs=460 cs=0 l=1 rev1=1 crypto rx=0x7f4808009e60 tx=0x7f4808009e90 comp rx=0 tx=0).stop 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1031824033 shutdown_connections 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a215f0 0x55c728a239e0 unknown :-1 s=CLOSED pgs=460 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a0eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1031824033 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a210b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1031824033 >> 172.21.15.90:0/1031824033 conn(0x55c72896d6a0 msgr2=0x55c728a09a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1031824033 shutdown_connections 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1031824033 wait complete. 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 -- start start 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 Processor -- start 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.268+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 0x55c728a1b110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a215f0 0x55c728a208f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c7288fbd80 con 0x55c728a0f0a0 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c7288fba20 con 0x55c728a215f0 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c7287e36a0 con 0x55c728a0e780 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 10 client.0 _ll_register_callbacks cb 0x55c728905470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:02.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45300/0 (socket says 172.21.15.90:45300) 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 -- 172.21.15.90:0/842528606 learned_addr learned my addr 172.21.15.90:0/842528606 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480effd700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 0x55c728a1b110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 -- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a215f0 msgr2=0x55c728a208f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a215f0 0x55c728a208f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 -- 172.21.15.90:0/842528606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 msgr2=0x55c728a1b110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 0x55c728a1b110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 -- 172.21.15.90:0/842528606 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4808009b10 con 0x55c728a0e780 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480effd700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 0x55c728a1b110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f480f7fe700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f4800005e90 tx=0x7f480009a9e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=955f3f0f6acf8dbe server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f47eeffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48000a1070 con 0x55c728a0e780 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c728920380 con 0x55c728a0e780 2024-04-06T14:06:02.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.269+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f480009ed80 con 0x55c728a0e780 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f48000a3340 con 0x55c728a0e780 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 10 client.25114 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c72878db60 con 0x55c728a0e780 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4800064080 con 0x55c728a0e780 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f48000a2eb0 con 0x55c728a0e780 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 10 client.25114 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 10 client.25114 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 20 client.25114 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a39730 0x55c728a3bb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 4 client.25114 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:02.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.270+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c728958700 con 0x55c728a39730 2024-04-06T14:06:02.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.271+0000 7f480effd700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a39730 0x55c728a3bb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.275+0000 7f47eeffd700 10 client.25114 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:02.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.275+0000 7f480effd700 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a39730 0x55c728a3bb20 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=37298bbbb484543a server_cookie=eb23032962c0e9d3 in_seq=0 out_seq=0 2024-04-06T14:06:02.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.276+0000 7f47eeffd700 1 -- 172.21.15.90:0/842528606 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c728958700 con 0x55c728a39730 2024-04-06T14:06:02.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.276+0000 7f47eeffd700 10 client.25114 handle_command_reply: tid=0 2024-04-06T14:06:02.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.276+0000 7f4821cf0b80 1 client.25114 shutdown 2024-04-06T14:06:02.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.277+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 msgr2=0x55c728a1abd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.277+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f4800005e90 tx=0x7f480009a9e0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 shutdown_connections 2024-04-06T14:06:02.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a215f0 0x55c728a208f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a39730 0x55c728a3bb20 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a0e780 0x55c728a1abd0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/842528606 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0f0a0 0x55c728a1b110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 >> 172.21.15.90:0/842528606 conn(0x55c72896d6a0 msgr2=0x55c728a08c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f47eeffd700 0 client.25114 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 shutdown_connections 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 1 -- 172.21.15.90:0/842528606 wait complete. 2024-04-06T14:06:02.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.278+0000 7f4821cf0b80 20 client.25114 trim_cache size 0 max 16384 2024-04-06T14:06:02.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.303+0000 7f4821cf0b80 1 Processor -- start 2024-04-06T14:06:02.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.303+0000 7f4821cf0b80 1 -- start start 2024-04-06T14:06:02.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a1a7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ad30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a14560 con 0x55c728a0e780 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a10630 con 0x55c728a0f0a0 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a10db0 con 0x55c728a215f0 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45316/0 (socket says 172.21.15.90:45316) 2024-04-06T14:06:02.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 -- 172.21.15.90:0/1423228809 learned_addr learned my addr 172.21.15.90:0/1423228809 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 -- 172.21.15.90:0/1423228809 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 msgr2=0x55c728a1ad30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 -- 172.21.15.90:0/1423228809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 msgr2=0x55c728a1a7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a1a7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.304+0000 7f480ffff700 1 -- 172.21.15.90:0/1423228809 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c728925ea0 con 0x55c728a215f0 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.305+0000 7f480ffff700 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4808018040 tx=0x7f4808004ab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fd7c90fb04362fb4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.305+0000 7f47f67fc700 1 -- 172.21.15.90:0/1423228809 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808011040 con 0x55c728a215f0 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.305+0000 7f47f67fc700 1 -- 172.21.15.90:0/1423228809 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4808009390 con 0x55c728a215f0 2024-04-06T14:06:02.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.305+0000 7f47f67fc700 1 -- 172.21.15.90:0/1423228809 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808015670 con 0x55c728a215f0 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1423228809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 msgr2=0x55c728a13230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4808018040 tx=0x7f4808004ab0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1423228809 shutdown_connections 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a13230 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a1a7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1423228809 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1423228809 >> 172.21.15.90:0/1423228809 conn(0x55c72896d6a0 msgr2=0x55c728a148a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1423228809 shutdown_connections 2024-04-06T14:06:02.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1423228809 wait complete. 2024-04-06T14:06:02.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.306+0000 7f4821cf0b80 1 -- start start 2024-04-06T14:06:02.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 Processor -- start 2024-04-06T14:06:02.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a20780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ec20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a1f8e0 con 0x55c728a0e780 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a1fa50 con 0x55c728a0f0a0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c728a8d420 con 0x55c728a215f0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.307+0000 7f4821cf0b80 10 client.0 _ll_register_callbacks cb 0x55c728905470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45322/0 (socket says 172.21.15.90:45322) 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 -- 172.21.15.90:0/1057552269 learned_addr learned my addr 172.21.15.90:0/1057552269 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480f7fe700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a20780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480effd700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 -- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 msgr2=0x55c728a1ec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ec20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 -- 172.21.15.90:0/1057552269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 msgr2=0x55c728a20780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a20780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 -- 172.21.15.90:0/1057552269 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c7287e13b0 con 0x55c728a215f0 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f480ffff700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4808020fc0 tx=0x7f4808009790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=af38a90ca4e50409 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.308+0000 7f47eeffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.309+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4808011040 con 0x55c728a215f0 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.309+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4808006e40 con 0x55c728a215f0 2024-04-06T14:06:02.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.309+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c7287aecf0 con 0x55c728a215f0 2024-04-06T14:06:02.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.312+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4808020870 con 0x55c728a215f0 2024-04-06T14:06:02.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 10 client.25120 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:02.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c7287e5cd0 con 0x55c728a215f0 2024-04-06T14:06:02.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c7287aecf0 con 0x55c728a215f0 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4808027030 con 0x55c728a215f0 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 10 client.25120 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 10 client.25120 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 20 client.25120 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.313+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a8e6f0 0x55c728a90b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.314+0000 7f4821cf0b80 4 client.25120 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:06:02.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.314+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c7287e3ce0 con 0x55c728a8e6f0 2024-04-06T14:06:02.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.317+0000 7f480f7fe700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a8e6f0 0x55c728a90b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.317+0000 7f47eeffd700 10 client.25120 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:02.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.317+0000 7f480f7fe700 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a8e6f0 0x55c728a90b10 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bd464a652e66412e server_cookie=2f15f2ce57988eca in_seq=0 out_seq=0 2024-04-06T14:06:02.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f47eeffd700 1 -- 172.21.15.90:0/1057552269 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55c7287e3ce0 con 0x55c728a8e6f0 2024-04-06T14:06:02.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f47eeffd700 10 client.25120 handle_command_reply: tid=0 2024-04-06T14:06:02.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 client.25120 shutdown 2024-04-06T14:06:02.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 msgr2=0x55c728a1f180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4808020fc0 tx=0x7f4808009790 comp rx=0 tx=0).stop 2024-04-06T14:06:02.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 shutdown_connections 2024-04-06T14:06:02.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c728a8e6f0 0x55c728a90b10 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c728a215f0 0x55c728a1f180 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c728a0e780 0x55c728a20780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 --2- 172.21.15.90:0/1057552269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c728a0f0a0 0x55c728a1ec20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 >> 172.21.15.90:0/1057552269 conn(0x55c72896d6a0 msgr2=0x55c728a0d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f47eeffd700 0 client.25120 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.318+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 shutdown_connections 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.319+0000 7f4821cf0b80 1 -- 172.21.15.90:0/1057552269 wait complete. 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.319+0000 7f4821cf0b80 20 client.25120 trim_cache size 0 max 16384 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:02.320 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.320+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 msgr2=0x7f47f80a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.320+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 0x7f47f80a38f0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f4804005950 tx=0x7f48040058e0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.320+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 msgr2=0x7f48101961b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.320+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f4800009810 tx=0x7f480000fab0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 shutdown_connections 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f47f80a1430 0x7f47f80a38f0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f481019aa90 0x7f481019af10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48100ffe70 0x7f48101961b0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 --2- 172.21.15.90:0/2349243250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4810104f70 0x7f48101966f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 >> 172.21.15.90:0/2349243250 conn(0x7f4810076d50 msgr2=0x7f48100fdff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 shutdown_connections 2024-04-06T14:06:02.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.321+0000 7f4815b3c700 1 -- 172.21.15.90:0/2349243250 wait complete. 2024-04-06T14:06:02.341 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:06:02.341 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 -- 172.21.15.90:0/4248046673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc10d6a0 msgr2=0x7fd4cc10da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 --2- 172.21.15.90:0/4248046673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc10d6a0 0x7fd4cc10da80 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fd4bc007510 tx=0x7fd4bc007820 comp rx=0 tx=0).stop 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 -- 172.21.15.90:0/4248046673 shutdown_connections 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 --2- 172.21.15.90:0/4248046673 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc108770 0x7fd4cc1144f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 --2- 172.21.15.90:0/4248046673 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc10d6a0 0x7fd4cc10da80 secure :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fd4bc007510 tx=0x7fd4bc007820 comp rx=0 tx=0).stop 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 --2- 172.21.15.90:0/4248046673 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 0x7fd4cc108230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 -- 172.21.15.90:0/4248046673 >> 172.21.15.90:0/4248046673 conn(0x7fd4cc06c180 msgr2=0x7fd4cc06c590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 -- 172.21.15.90:0/4248046673 shutdown_connections 2024-04-06T14:06:02.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.533+0000 7fd4d240c700 1 -- 172.21.15.90:0/4248046673 wait complete. 2024-04-06T14:06:02.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.534+0000 7fd4d240c700 1 Processor -- start 2024-04-06T14:06:02.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.534+0000 7fd4d240c700 1 -- start start 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.534+0000 7fd4d240c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 0x7fd4cc19d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d240c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d240c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 0x7fd4cc1add90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d240c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4cc1ae360 con 0x7fd4cc107db0 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d240c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4cc1ae4d0 con 0x7fd4cc108770 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d240c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4cc1ae690 con 0x7fd4cc19e230 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39558/0 (socket says 172.21.15.90:39558) 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4cb7fe700 1 -- 172.21.15.90:0/3833017780 learned_addr learned my addr 172.21.15.90:0/3833017780 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 0x7fd4cc1add90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4cbfff700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 0x7fd4cc19d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 -- 172.21.15.90:0/3833017780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 msgr2=0x7fd4cc19dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 -- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 msgr2=0x7fd4cc19d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 0x7fd4cc19d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4d09a9700 1 -- 172.21.15.90:0/3833017780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4bc0071c0 con 0x7fd4cc19e230 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.535+0000 7fd4cb7fe700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4d09a9700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 0x7fd4cc1add90 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4007f10 tx=0x7fd4c400d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4c400ddd0 con 0x7fd4cc19e230 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd4c400f040 con 0x7fd4cc19e230 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd4c40044b0 con 0x7fd4cc19e230 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4cc1ae970 con 0x7fd4cc19e230 2024-04-06T14:06:02.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.536+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4cc1aee30 con 0x7fd4cc19e230 2024-04-06T14:06:02.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.537+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd4b80052f0 con 0x7fd4cc19e230 2024-04-06T14:06:02.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.540+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 23) v1 ==== 136100+0+0 (secure 0 0 0) 0x7fd4c40094d0 con 0x7fd4cc19e230 2024-04-06T14:06:02.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.541+0000 7fd4c97fa700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 0x7fd4b40a3a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:02.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.541+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fd4c40cd160 con 0x7fd4cc19e230 2024-04-06T14:06:02.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.541+0000 7fd4cbfff700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 0x7fd4b40a3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:02.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.542+0000 7fd4cbfff700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 0x7fd4b40a3a40 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fd4bc00a040 tx=0x7fd4bc0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:02.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.543+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd4c4099e40 con 0x7fd4cc19e230 2024-04-06T14:06:02.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.690+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fd4c4096780 con 0x7fd4cc19e230 2024-04-06T14:06:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.951+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd4b8005160 con 0x7fd4cc19e230 2024-04-06T14:06:02.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.953+0000 7fd4c97fa700 1 -- 172.21.15.90:0/3833017780 <== mon.1 v2:172.21.15.134:3300/0 8 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fd4c4099bb0 con 0x7fd4cc19e230 2024-04-06T14:06:02.953 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:06:02.953 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:06:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 msgr2=0x7fd4b40a3a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 0x7fd4b40a3a40 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fd4bc00a040 tx=0x7fd4bc0058e0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 msgr2=0x7fd4cc1add90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 0x7fd4cc1add90 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4007f10 tx=0x7fd4c400d3b0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 shutdown_connections 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4b40a1580 0x7fd4b40a3a40 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4cc19e230 0x7fd4cc1add90 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd4cc107db0 0x7fd4cc19d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 --2- 172.21.15.90:0/3833017780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd4cc108770 0x7fd4cc19dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 >> 172.21.15.90:0/3833017780 conn(0x7fd4cc06c180 msgr2=0x7fd4cc071930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 shutdown_connections 2024-04-06T14:06:02.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:02.956+0000 7fd4d240c700 1 -- 172.21.15.90:0/3833017780 wait complete. 2024-04-06T14:06:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:06:02.975 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:02.976 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:06:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 15.0 s, 3781.2 tps, lat 26.283 ms stddev 50.956 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- 172.21.15.90:0/691701911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c069250 msgr2=0x7f2a8c069630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 --2- 172.21.15.90:0/691701911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c069250 0x7f2a8c069630 secure :-1 s=READY pgs=461 cs=0 l=1 rev1=1 crypto rx=0x7f2a80009990 tx=0x7f2a80009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- 172.21.15.90:0/691701911 shutdown_connections 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 --2- 172.21.15.90:0/691701911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 0x7f2a8c10dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 --2- 172.21.15.90:0/691701911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c069250 0x7f2a8c069630 unknown :-1 s=CLOSED pgs=461 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 --2- 172.21.15.90:0/691701911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069c00 0x7f2a8c105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- 172.21.15.90:0/691701911 >> 172.21.15.90:0/691701911 conn(0x7f2a8c0769b0 msgr2=0x7f2a8c076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- 172.21.15.90:0/691701911 shutdown_connections 2024-04-06T14:06:03.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- 172.21.15.90:0/691701911 wait complete. 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 Processor -- start 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.140+0000 7f2a92e39700 1 -- start start 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 0x7f2a8c197150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 0x7f2a8c197690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a8c19c4b0 con 0x7f2a8c19ba30 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a8c19c620 con 0x7f2a8c1060b0 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a92e39700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a8c19c7e0 con 0x7f2a8c069250 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52726/0 (socket says 172.21.15.90:52726) 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 -- 172.21.15.90:0/3376600127 learned_addr learned my addr 172.21.15.90:0/3376600127 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 0x7f2a8c197690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 0x7f2a8c197150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 -- 172.21.15.90:0/3376600127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 msgr2=0x7f2a8c197150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 0x7f2a8c197150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 -- 172.21.15.90:0/3376600127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 msgr2=0x7f2a8c197690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 0x7f2a8c197690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 -- 172.21.15.90:0/3376600127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a800095f0 con 0x7f2a8c19ba30 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 0x7f2a8c197150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.141+0000 7f2a913d6700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7f2a84007ee0 tx=0x7f2a8400d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.142+0000 7f2a89ffb700 1 -- 172.21.15.90:0/3376600127 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a84017070 con 0x7f2a8c19ba30 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.142+0000 7f2a89ffb700 1 -- 172.21.15.90:0/3376600127 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a8400f040 con 0x7f2a8c19ba30 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.142+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a8c1a3d40 con 0x7f2a8c19ba30 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.142+0000 7f2a89ffb700 1 -- 172.21.15.90:0/3376600127 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a84005740 con 0x7f2a8c19ba30 2024-04-06T14:06:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.142+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a8c1a41d0 con 0x7f2a8c19ba30 2024-04-06T14:06:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.144+0000 7f2a89ffb700 1 -- 172.21.15.90:0/3376600127 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f2a840058a0 con 0x7f2a8c19ba30 2024-04-06T14:06:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.145+0000 7f2a89ffb700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 0x7f2a7c0a3900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.145+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 0x7f2a7c0a3900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.145+0000 7f2a89ffb700 1 -- 172.21.15.90:0/3376600127 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2a840cc7d0 con 0x7f2a8c19ba30 2024-04-06T14:06:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 0x7f2a7c0a3900 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f2a8000b5c0 tx=0x7f2a80005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 Processor -- start 2024-04-06T14:06:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 -- start start 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f688a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877f6b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877eb6990 con 0x561877f5e780 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.146+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877eb4350 con 0x561877f5f0a0 2024-04-06T14:06:03.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877eafc40 con 0x561877f68de0 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52738/0 (socket says 172.21.15.90:52738) 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 -- 172.21.15.90:0/154927115 learned_addr learned my addr 172.21.15.90:0/154927115 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a913d6700 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877f6b1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f688a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 -- 172.21.15.90:0/154927115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 msgr2=0x561877f6b1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877f6b1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 -- 172.21.15.90:0/154927115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 msgr2=0x561877f688a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f688a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 -- 172.21.15.90:0/154927115 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561877d313b0 con 0x561877f5e780 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f2a80005b40 tx=0x7f2a8001e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3e9f4d4b1e9abddc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.147+0000 7f2a76ffd700 1 -- 172.21.15.90:0/154927115 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a8001e070 con 0x561877f5e780 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a76ffd700 1 -- 172.21.15.90:0/154927115 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a800055e0 con 0x561877f5e780 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a76ffd700 1 -- 172.21.15.90:0/154927115 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a80030480 con 0x561877f5e780 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 -- 172.21.15.90:0/154927115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 msgr2=0x561877f5eb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 crypto rx=0x7f2a80005b40 tx=0x7f2a8001e000 comp rx=0 tx=0).stop 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 -- 172.21.15.90:0/154927115 shutdown_connections 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877f6b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5e780 0x561877f5eb60 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/154927115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f688a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 -- 172.21.15.90:0/154927115 >> 172.21.15.90:0/154927115 conn(0x561877ebd6a0 msgr2=0x561877f59a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 -- 172.21.15.90:0/154927115 shutdown_connections 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.148+0000 7f2a9efedb80 1 -- 172.21.15.90:0/154927115 wait complete. 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 -- start start 2024-04-06T14:06:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 Processor -- start 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 0x561877f688a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f65c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877e4bd80 con 0x561877f68de0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877e4ba20 con 0x561877f5f0a0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877d336a0 con 0x561877f5e780 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a9efedb80 10 client.0 _ll_register_callbacks cb 0x561877e55470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52752/0 (socket says 172.21.15.90:52752) 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a913d6700 1 -- 172.21.15.90:0/2389483650 learned_addr learned my addr 172.21.15.90:0/2389483650 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f65c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.149+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 0x561877f688a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 -- 172.21.15.90:0/2389483650 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 msgr2=0x561877f688a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 0x561877f688a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 -- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 msgr2=0x561877f65c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f65c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 -- 172.21.15.90:0/2389483650 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a8c10e4a0 con 0x561877f68de0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f65c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 0x561877f688a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a913d6700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f2a84099640 tx=0x7f2a84099e60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6cd923f9142e88df server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a8409d320 con 0x561877f68de0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x561877e70380 con 0x561877f68de0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2a840a1b30 con 0x561877f68de0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2a84099910 con 0x561877f68de0 2024-04-06T14:06:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 10 client.15555 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561877cddb60 con 0x561877f68de0 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x561877e70380 con 0x561877f68de0 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2a840b4080 con 0x561877f68de0 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 10 client.15555 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 10 client.15555 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 20 client.15555 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877f78a20 0x561877f7ae40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 4 client.15555 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.150+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561877ea8700 con 0x561877f78a20 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.151+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877f78a20 0x561877f7ae40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.151+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877f78a20 0x561877f7ae40 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=6c2864296eefe7af server_cookie=7e28cc7cd870dcc9 in_seq=0 out_seq=0 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.151+0000 7f2a6bfff700 10 client.15555 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a6bfff700 1 -- 172.21.15.90:0/2389483650 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x561877ea8700 con 0x561877f78a20 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a6bfff700 10 client.15555 handle_command_reply: tid=0 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 client.15555 shutdown 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 msgr2=0x561877f663d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 secure :-1 s=READY pgs=464 cs=0 l=1 rev1=1 crypto rx=0x7f2a84099640 tx=0x7f2a84099e60 comp rx=0 tx=0).stop 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 shutdown_connections 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877f78a20 0x561877f7ae40 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f663d0 unknown :-1 s=CLOSED pgs=464 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5e780 0x561877f688a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/2389483650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5f0a0 0x561877f65c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a6bfff700 0 client.15555 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 >> 172.21.15.90:0/2389483650 conn(0x561877ebd6a0 msgr2=0x561877f58c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 shutdown_connections 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 1 -- 172.21.15.90:0/2389483650 wait complete. 2024-04-06T14:06:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.152+0000 7f2a9efedb80 20 client.15555 trim_cache size 0 max 16384 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 Processor -- start 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 -- start start 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f59c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5f0a0 0x561877f65570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877f5fa10 con 0x561877f68de0 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877f65ff0 con 0x561877f5e780 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877f60a90 con 0x561877f5f0a0 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a913d6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52768/0 (socket says 172.21.15.90:52768) 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a913d6700 1 -- 172.21.15.90:0/1560798343 learned_addr learned my addr 172.21.15.90:0/1560798343 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.165+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f59c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5f0a0 0x561877f65570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 -- 172.21.15.90:0/1560798343 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5f0a0 msgr2=0x561877f65570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5f0a0 0x561877f65570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 -- 172.21.15.90:0/1560798343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 msgr2=0x561877f59c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f59c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 -- 172.21.15.90:0/1560798343 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561877e75ea0 con 0x561877f68de0 2024-04-06T14:06:03.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a913d6700 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f2a84097a00 tx=0x7f2a840a19e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e4029b8dbe5e1b6b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a777fe700 1 -- 172.21.15.90:0/1560798343 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a8409d320 con 0x561877f68de0 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a777fe700 1 -- 172.21.15.90:0/1560798343 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a840a1b30 con 0x561877f68de0 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.166+0000 7f2a777fe700 1 -- 172.21.15.90:0/1560798343 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a84099b70 con 0x561877f68de0 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- 172.21.15.90:0/1560798343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 msgr2=0x561877f65ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 crypto rx=0x7f2a84097a00 tx=0x7f2a840a19e0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- 172.21.15.90:0/1560798343 shutdown_connections 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f68de0 0x561877f65ab0 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f59c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/1560798343 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f5f0a0 0x561877f65570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- 172.21.15.90:0/1560798343 >> 172.21.15.90:0/1560798343 conn(0x561877ebd6a0 msgr2=0x561877f59520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- 172.21.15.90:0/1560798343 shutdown_connections 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- 172.21.15.90:0/1560798343 wait complete. 2024-04-06T14:06:03.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 -- start start 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.167+0000 7f2a9efedb80 1 Processor -- start 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.168+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f6dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 0x561877f6e110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877fc9ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877fca210 con 0x561877f5f0a0 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877fca380 con 0x561877f5e780 2024-04-06T14:06:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561877fca540 con 0x561877f68de0 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a9efedb80 10 client.0 _ll_register_callbacks cb 0x561877e55470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a8bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 0x561877f6e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a8bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 0x561877f6e110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52770/0 (socket says 172.21.15.90:52770) 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a8bfff700 1 -- 172.21.15.90:0/666486704 learned_addr learned my addr 172.21.15.90:0/666486704 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f6dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a913d6700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877fc9ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a90bd5700 1 -- 172.21.15.90:0/666486704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 msgr2=0x561877fc9ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877fc9ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a90bd5700 1 -- 172.21.15.90:0/666486704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 msgr2=0x561877f6e110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.169+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 0x561877f6e110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a90bd5700 1 -- 172.21.15.90:0/666486704 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561877cddb60 con 0x561877f5e780 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a913d6700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877fc9ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:06:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a90bd5700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f6dbd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x561877f67190 tx=0x7f2a8000bd40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e032eb68dd323dd0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a6bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a80036040 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2a80004be0 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2a800055e0 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.170+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2a8001e040 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 10 client.25211 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561877d35cd0 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a800055e0 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2a80040070 con 0x561877f5e780 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 10 client.25211 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 10 client.25211 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 20 client.25211 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877fe2940 0x561877fe4d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 4 client.25211 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x561877cfecf0 con 0x561877fe2940 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.171+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877fe2940 0x561877fe4d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a6bfff700 10 client.25211 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a8bfff700 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877fe2940 0x561877fe4d30 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=c365906474428378 server_cookie=40852a5dcb2d604e in_seq=0 out_seq=0 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a6bfff700 1 -- 172.21.15.90:0/666486704 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x561877cfecf0 con 0x561877fe2940 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a6bfff700 10 client.25211 handle_command_reply: tid=0 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 client.25211 shutdown 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 msgr2=0x561877f6dbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f6dbd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x561877f67190 tx=0x7f2a8000bd40 comp rx=0 tx=0).stop 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 shutdown_connections 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561877fe2940 0x561877fe4d30 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561877f68de0 0x561877fc9ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561877f5e780 0x561877f6dbd0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a6bfff700 0 client.25211 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 --2- 172.21.15.90:0/666486704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561877f5f0a0 0x561877f6e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.172+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 >> 172.21.15.90:0/666486704 conn(0x561877ebd6a0 msgr2=0x561877f5d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.173+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 shutdown_connections 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.173+0000 7f2a9efedb80 1 -- 172.21.15.90:0/666486704 wait complete. 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.173+0000 7f2a9efedb80 20 client.25211 trim_cache size 0 max 16384 2024-04-06T14:06:03.178 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 msgr2=0x7f2a7c0a3900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 0x7f2a7c0a3900 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f2a8000b5c0 tx=0x7f2a80005fb0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 msgr2=0x7f2a8c19beb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 crypto rx=0x7f2a84007ee0 tx=0x7f2a8400d3b0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 shutdown_connections 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a7c0a1440 0x7f2a7c0a3900 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a8c19ba30 0x7f2a8c19beb0 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a8c069250 0x7f2a8c197150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 --2- 172.21.15.90:0/3376600127 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a8c1060b0 0x7f2a8c197690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.174+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 >> 172.21.15.90:0/3376600127 conn(0x7f2a8c0769b0 msgr2=0x7f2a8c0ff6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.175+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 shutdown_connections 2024-04-06T14:06:03.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.175+0000 7f2a92e39700 1 -- 172.21.15.90:0/3376600127 wait complete. 2024-04-06T14:06:03.195 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:03.195 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:06:03.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.403+0000 7f0d39f7c700 1 -- 172.21.15.90:0/2030171199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d340ffcc0 msgr2=0x7f0d34103960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.403+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/2030171199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d340ffcc0 0x7f0d34103960 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f0d2c00b0a0 tx=0x7f0d2c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.410+0000 7f0d39f7c700 1 -- 172.21.15.90:0/2030171199 shutdown_connections 2024-04-06T14:06:03.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.410+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/2030171199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d340ffcc0 0x7f0d34103960 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.410+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/2030171199 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d34105e60 0x7f0d34106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.410+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/2030171199 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 0x7f0d340ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.410+0000 7f0d39f7c700 1 -- 172.21.15.90:0/2030171199 >> 172.21.15.90:0/2030171199 conn(0x7f0d340761b0 msgr2=0x7f0d340fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.411+0000 7f0d39f7c700 1 -- 172.21.15.90:0/2030171199 shutdown_connections 2024-04-06T14:06:03.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.411+0000 7f0d39f7c700 1 -- 172.21.15.90:0/2030171199 wait complete. 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.412+0000 7f0d39f7c700 1 Processor -- start 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.412+0000 7f0d39f7c700 1 -- start start 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 0x7f0d3410ae80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 0x7f0d3410b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 0x7f0d34106970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d34107140 con 0x7f0d340ff220 2024-04-06T14:06:03.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d341072b0 con 0x7f0d340ffcc0 2024-04-06T14:06:03.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.413+0000 7f0d39f7c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d34107470 con 0x7f0d34105e60 2024-04-06T14:06:03.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.414+0000 7f0d33fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 0x7f0d34106970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.414+0000 7f0d33fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 0x7f0d34106970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46250/0 (socket says 172.21.15.90:46250) 2024-04-06T14:06:03.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.414+0000 7f0d33fff700 1 -- 172.21.15.90:0/1331039092 learned_addr learned my addr 172.21.15.90:0/1331039092 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.414+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 0x7f0d3410b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d337fe700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 0x7f0d3410ae80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d32ffd700 1 -- 172.21.15.90:0/1331039092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 msgr2=0x7f0d34106970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 0x7f0d34106970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d32ffd700 1 -- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 msgr2=0x7f0d3410ae80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 0x7f0d3410ae80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.415+0000 7f0d32ffd700 1 -- 172.21.15.90:0/1331039092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d2c009d00 con 0x7f0d340ffcc0 2024-04-06T14:06:03.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.416+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 0x7f0d3410b3e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f0d2800b450 tx=0x7f0d2800b810 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.416+0000 7f0d30ff9700 1 -- 172.21.15.90:0/1331039092 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d28011830 con 0x7f0d340ffcc0 2024-04-06T14:06:03.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.417+0000 7f0d30ff9700 1 -- 172.21.15.90:0/1331039092 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d28011e50 con 0x7f0d340ffcc0 2024-04-06T14:06:03.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.417+0000 7f0d30ff9700 1 -- 172.21.15.90:0/1331039092 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2800f2e0 con 0x7f0d340ffcc0 2024-04-06T14:06:03.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.418+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d34107750 con 0x7f0d340ffcc0 2024-04-06T14:06:03.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.418+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d34112eb0 con 0x7f0d340ffcc0 2024-04-06T14:06:03.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.422+0000 7f0d30ff9700 1 -- 172.21.15.90:0/1331039092 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f0d280105a0 con 0x7f0d340ffcc0 2024-04-06T14:06:03.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.424+0000 7f0d30ff9700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 0x7f0d1c0a3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.424+0000 7f0d30ff9700 1 -- 172.21.15.90:0/1331039092 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0d280cc8d0 con 0x7f0d340ffcc0 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 Processor -- start 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 -- start start 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecb6b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecc0de0 0x55bebecc31d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebec0e990 con 0x55bebecc0de0 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebec0c350 con 0x55bebecb6780 2024-04-06T14:06:03.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.426+0000 7f0d46130b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebec07c40 con 0x55bebecb70a0 2024-04-06T14:06:03.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46262/0 (socket says 172.21.15.90:46262) 2024-04-06T14:06:03.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 -- 172.21.15.90:0/161635874 learned_addr learned my addr 172.21.15.90:0/161635874 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 -- 172.21.15.90:0/161635874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 msgr2=0x55bebecb6b60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecb6b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 -- 172.21.15.90:0/161635874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecc0de0 msgr2=0x55bebecc31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecc0de0 0x55bebecc31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 -- 172.21.15.90:0/161635874 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bebea893b0 con 0x55bebecb70a0 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f0d280a1810 tx=0x7f0d28099480 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a574c62a952bbeb1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.427+0000 7f0d19ffb700 1 -- 172.21.15.90:0/161635874 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d280998c0 con 0x55bebecb70a0 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.428+0000 7f0d19ffb700 1 -- 172.21.15.90:0/161635874 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d2809d6f0 con 0x55bebecb70a0 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.428+0000 7f0d19ffb700 1 -- 172.21.15.90:0/161635874 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d280a1b10 con 0x55bebecb70a0 2024-04-06T14:06:03.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.428+0000 7f0d337fe700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 0x7f0d1c0a3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 -- 172.21.15.90:0/161635874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 msgr2=0x55bebecc08a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f0d280a1810 tx=0x7f0d28099480 comp rx=0 tx=0).stop 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 -- 172.21.15.90:0/161635874 shutdown_connections 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecc0de0 0x55bebecc31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecb6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 --2- 172.21.15.90:0/161635874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb70a0 0x55bebecc08a0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.429+0000 7f0d46130b80 1 -- 172.21.15.90:0/161635874 >> 172.21.15.90:0/161635874 conn(0x55bebec156a0 msgr2=0x55bebecb1a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.430+0000 7f0d46130b80 1 -- 172.21.15.90:0/161635874 shutdown_connections 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.430+0000 7f0d46130b80 1 -- 172.21.15.90:0/161635874 wait complete. 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.430+0000 7f0d337fe700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 0x7f0d1c0a3a50 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f0d24005e20 tx=0x7f0d24005ed0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.430+0000 7f0d46130b80 1 -- start start 2024-04-06T14:06:03.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.430+0000 7f0d46130b80 1 Processor -- start 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecbdcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbe210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebecbe750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebeba3d80 con 0x55bebecb70a0 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebeba3a20 con 0x55bebecb6780 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebea8b6a0 con 0x55bebecc0de0 2024-04-06T14:06:03.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.431+0000 7f0d46130b80 10 client.0 _ll_register_callbacks cb 0x55bebebad470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbe210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbe210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54532/0 (socket says 172.21.15.90:54532) 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d32ffd700 1 -- 172.21.15.90:0/2401775492 learned_addr learned my addr 172.21.15.90:0/2401775492 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebecbe750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d337fe700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecbdcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 -- 172.21.15.90:0/2401775492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 msgr2=0x55bebecbdcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecbdcd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 -- 172.21.15.90:0/2401775492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 msgr2=0x55bebecbe210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbe210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.432+0000 7f0d33fff700 1 -- 172.21.15.90:0/2401775492 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d34112eb0 con 0x55bebecc0de0 2024-04-06T14:06:03.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebecbe750 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x55bebecb81f0 tx=0x7f0d2c0095b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c2f65fbcee387a75 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d12ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2c00e050 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0d2c0074e0 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bebebc8380 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d2c0076a0 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d46130b80 10 client.25150 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bebea35b60 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.433+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55bebebc8380 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0d2c00b0d0 con 0x55bebecc0de0 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 10 client.25150 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 10 client.25150 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 20 client.25150 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebecd0a60 0x55bebecd2e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 4 client.25150 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:03.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bebec00700 con 0x55bebecd0a60 2024-04-06T14:06:03.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d337fe700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebecd0a60 0x55bebecd2e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d12ffd700 10 client.25150 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.434+0000 7f0d337fe700 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebecd0a60 0x55bebecd2e80 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=bb4b7d75450831c6 server_cookie=a5bf063fa96d040f in_seq=0 out_seq=0 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.435+0000 7f0d12ffd700 1 -- 172.21.15.90:0/2401775492 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55bebec00700 con 0x55bebecd0a60 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.435+0000 7f0d12ffd700 10 client.25150 handle_command_reply: tid=0 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.435+0000 7f0d46130b80 1 client.25150 shutdown 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.435+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 msgr2=0x55bebecbe750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.435+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebecbe750 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x55bebecb81f0 tx=0x7f0d2c0095b0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 shutdown_connections 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebecd0a60 0x55bebecd2e80 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebecbe750 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb6780 0x55bebecbdcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2401775492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbe210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d12ffd700 0 client.25150 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 >> 172.21.15.90:0/2401775492 conn(0x55bebec156a0 msgr2=0x55bebecb0c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 shutdown_connections 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 1 -- 172.21.15.90:0/2401775492 wait complete. 2024-04-06T14:06:03.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.436+0000 7f0d46130b80 20 client.25150 trim_cache size 0 max 16384 2024-04-06T14:06:03.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 Processor -- start 2024-04-06T14:06:03.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 -- start start 2024-04-06T14:06:03.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb6780 0x55bebecd2e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbd590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebecb7a50 con 0x55bebecb70a0 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebecb7fe0 con 0x55bebecc0de0 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.457+0000 7f0d46130b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebecbdad0 con 0x55bebecb6780 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d337fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb6780 0x55bebecd2e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49348/0 (socket says 172.21.15.90:49348) 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 -- 172.21.15.90:0/2445371031 learned_addr learned my addr 172.21.15.90:0/2445371031 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbd590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 -- 172.21.15.90:0/2445371031 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb6780 msgr2=0x55bebecd2e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb6780 0x55bebecd2e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 -- 172.21.15.90:0/2445371031 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 msgr2=0x55bebecbd590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbd590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.458+0000 7f0d33fff700 1 -- 172.21.15.90:0/2445371031 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bebebcdea0 con 0x55bebecc0de0 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.459+0000 7f0d33fff700 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f0d2c004a50 tx=0x7f0d2c004a80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=313eebe5eee7984c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.459+0000 7f0d1a7fc700 1 -- 172.21.15.90:0/2445371031 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2c00e070 con 0x55bebecc0de0 2024-04-06T14:06:03.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.460+0000 7f0d1a7fc700 1 -- 172.21.15.90:0/2445371031 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d2c00f040 con 0x55bebecc0de0 2024-04-06T14:06:03.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.460+0000 7f0d1a7fc700 1 -- 172.21.15.90:0/2445371031 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d2c01bca0 con 0x55bebecc0de0 2024-04-06T14:06:03.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.461+0000 7f0d46130b80 1 -- 172.21.15.90:0/2445371031 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 msgr2=0x55bebecb5e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.461+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f0d2c004a50 tx=0x7f0d2c004a80 comp rx=0 tx=0).stop 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 -- 172.21.15.90:0/2445371031 shutdown_connections 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecc0de0 0x55bebecb5e80 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecb6780 0x55bebecd2e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 --2- 172.21.15.90:0/2445371031 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb70a0 0x55bebecbd590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 -- 172.21.15.90:0/2445371031 >> 172.21.15.90:0/2445371031 conn(0x55bebec156a0 msgr2=0x55bebecb5770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 -- 172.21.15.90:0/2445371031 shutdown_connections 2024-04-06T14:06:03.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.462+0000 7f0d46130b80 1 -- 172.21.15.90:0/2445371031 wait complete. 2024-04-06T14:06:03.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 -- start start 2024-04-06T14:06:03.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 Processor -- start 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb6780 0x55bebecc5c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebed23850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebed23d90 con 0x55bebecb6780 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebed23ed0 con 0x55bebecb70a0 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bebed24040 con 0x55bebecc0de0 2024-04-06T14:06:03.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.463+0000 7f0d46130b80 10 client.0 _ll_register_callbacks cb 0x55bebebad470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49354/0 (socket says 172.21.15.90:49354) 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 -- 172.21.15.90:0/400894569 learned_addr learned my addr 172.21.15.90:0/400894569 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d337fe700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb6780 0x55bebecc5c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d33fff700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebed23850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 -- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 msgr2=0x55bebed23850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebed23850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 -- 172.21.15.90:0/400894569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb6780 msgr2=0x55bebecc5c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb6780 0x55bebecc5c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.464+0000 7f0d32ffd700 1 -- 172.21.15.90:0/400894569 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55bebea35b60 con 0x55bebecb70a0 2024-04-06T14:06:03.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d32ffd700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f0d280093b0 tx=0x7f0d280998c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ab7ab595dc66fc72 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d12ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d280b2240 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0d28010380 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0d2809d6f0 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d28098fc0 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d46130b80 10 client.25223 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.465+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bebea8dcd0 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d2809d6f0 con 0x55bebecb70a0 2024-04-06T14:06:03.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0d28017b70 con 0x55bebecb70a0 2024-04-06T14:06:03.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 10 client.25223 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 10 client.25223 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 20 client.25223 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebed25440 0x55bebed27860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 4 client.25223 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.466+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55bebea56cf0 con 0x55bebed25440 2024-04-06T14:06:03.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d337fe700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebed25440 0x55bebed27860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d12ffd700 10 client.25223 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:03.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d337fe700 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebed25440 0x55bebed27860 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=b5a341270b8d225 server_cookie=9e9f44d2da444c3e in_seq=0 out_seq=0 2024-04-06T14:06:03.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d12ffd700 1 -- 172.21.15.90:0/400894569 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55bebea56cf0 con 0x55bebed25440 2024-04-06T14:06:03.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d12ffd700 10 client.25223 handle_command_reply: tid=0 2024-04-06T14:06:03.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d46130b80 1 client.25223 shutdown 2024-04-06T14:06:03.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 msgr2=0x55bebecc6170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.467+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f0d280093b0 tx=0x7f0d280998c0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 shutdown_connections 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bebed25440 0x55bebed27860 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bebecc0de0 0x55bebed23850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bebecb6780 0x55bebecc5c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d12ffd700 0 client.25223 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 --2- 172.21.15.90:0/400894569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bebecb70a0 0x55bebecc6170 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 >> 172.21.15.90:0/400894569 conn(0x55bebec156a0 msgr2=0x55bebecb6000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 shutdown_connections 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 1 -- 172.21.15.90:0/400894569 wait complete. 2024-04-06T14:06:03.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.469+0000 7f0d46130b80 20 client.25223 trim_cache size 0 max 16384 2024-04-06T14:06:03.473 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:03.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.471+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 msgr2=0x7f0d1c0a3a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.471+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 0x7f0d1c0a3a50 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f0d24005e20 tx=0x7f0d24005ed0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.471+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 msgr2=0x7f0d3410b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.471+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 0x7f0d3410b3e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f0d2800b450 tx=0x7f0d2800b810 comp rx=0 tx=0).stop 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 shutdown_connections 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d1c0a1590 0x7f0d1c0a3a50 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d34105e60 0x7f0d34106970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d340ff220 0x7f0d3410ae80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 --2- 172.21.15.90:0/1331039092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d340ffcc0 0x7f0d3410b3e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 >> 172.21.15.90:0/1331039092 conn(0x7f0d340761b0 msgr2=0x7f0d341014c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 shutdown_connections 2024-04-06T14:06:03.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.472+0000 7f0d39f7c700 1 -- 172.21.15.90:0/1331039092 wait complete. 2024-04-06T14:06:03.492 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:03.493 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:06:03.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 -- 172.21.15.90:0/3287566121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 msgr2=0x7fefb810fd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 --2- 172.21.15.90:0/3287566121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb810fd90 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 crypto rx=0x7fefa8009990 tx=0x7fefa8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 -- 172.21.15.90:0/3287566121 shutdown_connections 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 --2- 172.21.15.90:0/3287566121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb810fd90 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 --2- 172.21.15.90:0/3287566121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb80ff220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 --2- 172.21.15.90:0/3287566121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefb80ff760 0x7fefb80ffbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 -- 172.21.15.90:0/3287566121 >> 172.21.15.90:0/3287566121 conn(0x7fefb80761b0 msgr2=0x7fefb80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 -- 172.21.15.90:0/3287566121 shutdown_connections 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.670+0000 7fefbdce8700 1 -- 172.21.15.90:0/3287566121 wait complete. 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.671+0000 7fefbdce8700 1 Processor -- start 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.671+0000 7fefbdce8700 1 -- start start 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefb80ff760 0x7fefb81995f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb8199b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb819ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefb819e510 con 0x7fefb810d9a0 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefb819e680 con 0x7fefb8101e20 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefbdce8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefb819e840 con 0x7fefb80ff760 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb819ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb819ded0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54590/0 (socket says 172.21.15.90:54590) 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb7fff700 1 -- 172.21.15.90:0/2208508104 learned_addr learned my addr 172.21.15.90:0/2208508104 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb8199b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 -- 172.21.15.90:0/2208508104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefb80ff760 msgr2=0x7fefb81995f0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:06:03.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefb80ff760 0x7fefb81995f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 -- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 msgr2=0x7fefb819ded0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb819ded0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 -- 172.21.15.90:0/2208508104 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fefa80095f0 con 0x7fefb8101e20 2024-04-06T14:06:03.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.672+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb8199b30 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fefb000ef10 tx=0x7fefb000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.673+0000 7fefb4ff9700 1 -- 172.21.15.90:0/2208508104 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb000ca70 con 0x7fefb8101e20 2024-04-06T14:06:03.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.673+0000 7fefb4ff9700 1 -- 172.21.15.90:0/2208508104 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefb000f040 con 0x7fefb8101e20 2024-04-06T14:06:03.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.673+0000 7fefb4ff9700 1 -- 172.21.15.90:0/2208508104 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb00146e0 con 0x7fefb8101e20 2024-04-06T14:06:03.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.675+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fefb819eac0 con 0x7fefb8101e20 2024-04-06T14:06:03.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.675+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefb81a6360 con 0x7fefb8101e20 2024-04-06T14:06:03.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.679+0000 7fefc9e9cb80 1 Processor -- start 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 -- start start 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6e8b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f51d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a640990 con 0x55692a6e8780 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a63e350 con 0x55692a6f2de0 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a639c40 con 0x55692a6e90a0 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46292/0 (socket says 172.21.15.90:46292) 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 -- 172.21.15.90:0/2355920952 learned_addr learned my addr 172.21.15.90:0/2355920952 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 -- 172.21.15.90:0/2355920952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 msgr2=0x55692a6f51d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f51d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 -- 172.21.15.90:0/2355920952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 msgr2=0x55692a6e8b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6e8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.680+0000 7fefb77fe700 1 -- 172.21.15.90:0/2355920952 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55692a4bb3b0 con 0x55692a6e90a0 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.681+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fefac009be0 tx=0x7fefac009ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9f8662f861803482 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.681+0000 7fef9dffb700 1 -- 172.21.15.90:0/2355920952 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefac00eb30 con 0x55692a6e90a0 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.681+0000 7fef9dffb700 1 -- 172.21.15.90:0/2355920952 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefac004cf0 con 0x55692a6e90a0 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.681+0000 7fef9dffb700 1 -- 172.21.15.90:0/2355920952 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefac005680 con 0x55692a6e90a0 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.681+0000 7fefb4ff9700 1 -- 172.21.15.90:0/2208508104 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fefb0007500 con 0x7fefb8101e20 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.682+0000 7fefb4ff9700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 0x7fefa00a3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.682+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2355920952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 msgr2=0x55692a6f28a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.682+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7fefac009be0 tx=0x7fefac009ef0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.682+0000 7fefb4ff9700 1 -- 172.21.15.90:0/2208508104 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fefb00ccc50 con 0x7fefb8101e20 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2355920952 shutdown_connections 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f51d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6e8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2355920952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6f28a0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2355920952 >> 172.21.15.90:0/2355920952 conn(0x55692a6476a0 msgr2=0x55692a6e3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.683+0000 7fefb7fff700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 0x7fefa00a3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.684+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2355920952 shutdown_connections 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.684+0000 7fefb7fff700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 0x7fefa00a3a50 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fefa8005950 tx=0x7fefa80058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.684+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2355920952 wait complete. 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.684+0000 7fefc9e9cb80 1 -- start start 2024-04-06T14:06:03.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 Processor -- start 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6efbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f02f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a5d5d80 con 0x55692a6e8780 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a5d5a20 con 0x55692a6f2de0 2024-04-06T14:06:03.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a4bd6a0 con 0x55692a6e90a0 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefc9e9cb80 10 client.0 _ll_register_callbacks cb 0x55692a5df470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54610/0 (socket says 172.21.15.90:54610) 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.685+0000 7fefb77fe700 1 -- 172.21.15.90:0/3581808550 learned_addr learned my addr 172.21.15.90:0/3581808550 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb7fff700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f02f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6efbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 -- 172.21.15.90:0/3581808550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 msgr2=0x55692a6efbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6efbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 -- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 msgr2=0x55692a6f02f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f02f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 -- 172.21.15.90:0/3581808550 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefac009890 con 0x55692a6e8780 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6efbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb7fff700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fefb77fe700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fefac0052a0 tx=0x7fefac004980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2088cf6a69afd74f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.686+0000 7fef96ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.689+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefac009730 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.689+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55692a5fa380 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.689+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fefac014070 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fefac004e60 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 10 client.15582 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55692a467b60 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55692a5fa380 con 0x55692a6e8780 2024-04-06T14:06:03.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fefac003680 con 0x55692a6e8780 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 10 client.15582 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 10 client.15582 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 20 client.15582 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7028c0 0x55692a704cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 4 client.15582 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55692a632700 con 0x55692a7028c0 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.690+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7028c0 0x55692a704cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.691+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7028c0 0x55692a704cb0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bf2c82b109c16d19 server_cookie=2ea6c2f451f82b54 in_seq=0 out_seq=0 2024-04-06T14:06:03.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.691+0000 7fef96ffd700 10 client.15582 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.691+0000 7fef96ffd700 1 -- 172.21.15.90:0/3581808550 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55692a632700 con 0x55692a7028c0 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.691+0000 7fef96ffd700 10 client.15582 handle_command_reply: tid=0 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.691+0000 7fefc9e9cb80 1 client.15582 shutdown 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.692+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 msgr2=0x55692a6f2820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.692+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 secure :-1 s=READY pgs=467 cs=0 l=1 rev1=1 crypto rx=0x7fefac0052a0 tx=0x7fefac004980 comp rx=0 tx=0).stop 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 shutdown_connections 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6f02f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e8780 0x55692a6f2820 unknown :-1 s=CLOSED pgs=467 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7028c0 0x55692a704cb0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/3581808550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e90a0 0x55692a6efbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 >> 172.21.15.90:0/3581808550 conn(0x55692a6476a0 msgr2=0x55692a6e2c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fef96ffd700 0 client.15582 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 shutdown_connections 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/3581808550 wait complete. 2024-04-06T14:06:03.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.693+0000 7fefc9e9cb80 20 client.15582 trim_cache size 0 max 16384 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 Processor -- start 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 -- start start 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6ea630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6eb420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6ea1b0 con 0x55692a6e90a0 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6e5180 con 0x55692a6f2de0 2024-04-06T14:06:03.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6ee140 con 0x55692a6e8780 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6eb420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6eb420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49392/0 (socket says 172.21.15.90:49392) 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54622/0 (socket says 172.21.15.90:54622) 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6ea630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.711+0000 7fefb7fff700 1 -- 172.21.15.90:0/899435321 learned_addr learned my addr 172.21.15.90:0/899435321 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 -- 172.21.15.90:0/899435321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 msgr2=0x55692a6ea630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6ea630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 -- 172.21.15.90:0/899435321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 msgr2=0x55692a6eb420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6eb420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 -- 172.21.15.90:0/899435321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55692a5ffea0 con 0x55692a6e90a0 2024-04-06T14:06:03.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.712+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fefb00ca170 tx=0x7fefb0009eb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1631a63bd26b19c3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.713+0000 7fef9e7fc700 1 -- 172.21.15.90:0/899435321 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb0007e40 con 0x55692a6e90a0 2024-04-06T14:06:03.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.713+0000 7fef9e7fc700 1 -- 172.21.15.90:0/899435321 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefb00112c0 con 0x55692a6e90a0 2024-04-06T14:06:03.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.713+0000 7fef9e7fc700 1 -- 172.21.15.90:0/899435321 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefb0099e20 con 0x55692a6e90a0 2024-04-06T14:06:03.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.713+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/899435321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 msgr2=0x55692a6eaee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.713+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 secure :-1 s=READY pgs=468 cs=0 l=1 rev1=1 crypto rx=0x7fefb00ca170 tx=0x7fefb0009eb0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.714+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/899435321 shutdown_connections 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.714+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6f2de0 0x55692a6eb420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.714+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6ea630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.714+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/899435321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6e90a0 0x55692a6eaee0 unknown :-1 s=CLOSED pgs=468 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.714+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/899435321 >> 172.21.15.90:0/899435321 conn(0x55692a6476a0 msgr2=0x55692a709630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.715+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/899435321 shutdown_connections 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.715+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/899435321 wait complete. 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.715+0000 7fefc9e9cb80 1 -- start start 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.715+0000 7fefc9e9cb80 1 Processor -- start 2024-04-06T14:06:03.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6e90a0 0x55692a6f7af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6f2de0 0x55692a6f8030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6fa060 con 0x55692a6f2de0 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6f85a0 con 0x55692a6e90a0 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55692a6f8760 con 0x55692a6e8780 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefc9e9cb80 10 client.0 _ll_register_callbacks cb 0x55692a5df470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefb77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46322/0 (socket says 172.21.15.90:46322) 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefb77fe700 1 -- 172.21.15.90:0/2017182125 learned_addr learned my addr 172.21.15.90:0/2017182125 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:03.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.716+0000 7fefb7fff700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6f2de0 0x55692a6f8030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 -- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6e90a0 msgr2=0x55692a6f7af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6e90a0 0x55692a6f7af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 -- 172.21.15.90:0/2017182125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6f2de0 msgr2=0x55692a6f8030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6f2de0 0x55692a6f8030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 -- 172.21.15.90:0/2017182125 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefb81a6360 con 0x55692a6e8780 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fefb77fe700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fefac003a90 tx=0x7fefac009350 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c207e96022b8ed44 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:03.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fef96ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefac005060 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.717+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fefac014040 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fefac004cf0 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fefac00e930 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 10 client.25168 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55692a4bfcd0 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefac044030 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fefac004410 con 0x55692a6e8780 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 10 client.25168 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 10 client.25168 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 20 client.25168 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7577b0 0x55692a759bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 4 client.25168 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:03.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.718+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55692a488cf0 con 0x55692a7577b0 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7577b0 0x55692a759bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fefb6ffd700 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7577b0 0x55692a759bd0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=81cb24e5f89c6b84 server_cookie=2716b7ba6c9636ce in_seq=0 out_seq=0 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fef96ffd700 10 client.25168 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fef96ffd700 1 -- 172.21.15.90:0/2017182125 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55692a488cf0 con 0x55692a7577b0 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fef96ffd700 10 client.25168 handle_command_reply: tid=0 2024-04-06T14:06:03.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.719+0000 7fefc9e9cb80 1 client.25168 shutdown 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 msgr2=0x55692a6f9750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fefac003a90 tx=0x7fefac009350 comp rx=0 tx=0).stop 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 shutdown_connections 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55692a6f2de0 0x55692a6f8030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55692a7577b0 0x55692a759bd0 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55692a6e8780 0x55692a6f9750 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 --2- 172.21.15.90:0/2017182125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55692a6e90a0 0x55692a6f7af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 >> 172.21.15.90:0/2017182125 conn(0x55692a6476a0 msgr2=0x55692a6e7770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.720+0000 7fef96ffd700 0 client.25168 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:03.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.721+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 shutdown_connections 2024-04-06T14:06:03.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.721+0000 7fefc9e9cb80 1 -- 172.21.15.90:0/2017182125 wait complete. 2024-04-06T14:06:03.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.721+0000 7fefc9e9cb80 20 client.25168 trim_cache size 0 max 16384 2024-04-06T14:06:03.724 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 msgr2=0x7fefa00a3a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 0x7fefa00a3a50 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fefa8005950 tx=0x7fefa80058e0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 msgr2=0x7fefb8199b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb8199b30 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fefb000ef10 tx=0x7fefb000c3a0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 shutdown_connections 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefa00a1590 0x7fefa00a3a50 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefb810d9a0 0x7fefb819ded0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefb80ff760 0x7fefb81995f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 --2- 172.21.15.90:0/2208508104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefb8101e20 0x7fefb8199b30 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.724+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 >> 172.21.15.90:0/2208508104 conn(0x7fefb80761b0 msgr2=0x7fefb810ee40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:03.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.725+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 shutdown_connections 2024-04-06T14:06:03.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:03.725+0000 7fefbdce8700 1 -- 172.21.15.90:0/2208508104 wait complete. 2024-04-06T14:06:03.745 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:03 smithi090 ceph-mon[43420]: mgrmap e24: x(active, since 39s), standbys: z, y 2024-04-06T14:06:03.745 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:03 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3833017780' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:03.745 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:03 smithi090 ceph-mon[43420]: pgmap v23: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 85 MiB/s wr, 1.03k op/s 2024-04-06T14:06:03.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:03 smithi105 ceph-mon[46969]: mgrmap e24: x(active, since 39s), standbys: z, y 2024-04-06T14:06:03.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:03 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3833017780' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:03.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:03 smithi105 ceph-mon[46969]: pgmap v23: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 85 MiB/s wr, 1.03k op/s 2024-04-06T14:06:04.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:03 smithi134 ceph-mon[47549]: mgrmap e24: x(active, since 39s), standbys: z, y 2024-04-06T14:06:04.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:03 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3833017780' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:04.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:03 smithi134 ceph-mon[47549]: pgmap v23: 97 pgs: 97 active+clean; 1.2 GiB data, 1.2 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 85 MiB/s wr, 1.03k op/s 2024-04-06T14:06:04.745 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:06:04.745 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:06:04.745 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:06:04.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 -- 172.21.15.90:0/3768666805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c0ffe60 msgr2=0x7fb68c100240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 --2- 172.21.15.90:0/3768666805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c0ffe60 0x7fb68c100240 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fb6800098f0 tx=0x7fb680009c00 comp rx=0 tx=0).stop 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 -- 172.21.15.90:0/3768666805 shutdown_connections 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 --2- 172.21.15.90:0/3768666805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c10cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 --2- 172.21.15.90:0/3768666805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c0ffe60 0x7fb68c100240 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 --2- 172.21.15.90:0/3768666805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c100810 0x7fb68c104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 -- 172.21.15.90:0/3768666805 >> 172.21.15.90:0/3768666805 conn(0x7fb68c076db0 msgr2=0x7fb68c0771c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 -- 172.21.15.90:0/3768666805 shutdown_connections 2024-04-06T14:06:04.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.945+0000 7fb690977700 1 -- 172.21.15.90:0/3768666805 wait complete. 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 Processor -- start 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 -- start start 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c1070e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 0x7fb68c1a54a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb68c1a5a70 con 0x7fb68c0ffe60 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.946+0000 7fb690977700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb68c1a5be0 con 0x7fb68c104f80 2024-04-06T14:06:04.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb690977700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb68c1a5da0 con 0x7fb68c107620 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb689d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c1070e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54670/0 (socket says 172.21.15.90:54670) 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68a59c700 1 -- 172.21.15.90:0/318931829 learned_addr learned my addr 172.21.15.90:0/318931829 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb689d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c1070e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49416/0 (socket says 172.21.15.90:49416) 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 0x7fb68c1a54a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 -- 172.21.15.90:0/318931829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 msgr2=0x7fb68c1070e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c1070e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 -- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 msgr2=0x7fb68c106ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68ad9d700 1 -- 172.21.15.90:0/318931829 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb684009d00 con 0x7fb68c107620 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.947+0000 7fb68a59c700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.948+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 0x7fb68c1a54a0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fb67c00e970 tx=0x7fb67c00ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.948+0000 7fb67b7fe700 1 -- 172.21.15.90:0/318931829 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb67c004220 con 0x7fb68c107620 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.948+0000 7fb67b7fe700 1 -- 172.21.15.90:0/318931829 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb67c004cf0 con 0x7fb68c107620 2024-04-06T14:06:04.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.948+0000 7fb67b7fe700 1 -- 172.21.15.90:0/318931829 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb67c005660 con 0x7fb68c107620 2024-04-06T14:06:04.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.949+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb680009520 con 0x7fb68c107620 2024-04-06T14:06:04.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.949+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb68c197440 con 0x7fb68c107620 2024-04-06T14:06:04.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.952+0000 7fb67b7fe700 1 -- 172.21.15.90:0/318931829 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fb67c004750 con 0x7fb68c107620 2024-04-06T14:06:04.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 Processor -- start 2024-04-06T14:06:04.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 -- start start 2024-04-06T14:06:04.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108214800 0x557108214be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108215120 0x55710821e920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710816ca10 con 0x557108214800 2024-04-06T14:06:04.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710816a3d0 con 0x557108215120 2024-04-06T14:06:04.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.954+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557108165cc0 con 0x55710821ee60 2024-04-06T14:06:04.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46338/0 (socket says 172.21.15.90:46338) 2024-04-06T14:06:04.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 -- 172.21.15.90:0/1849918612 learned_addr learned my addr 172.21.15.90:0/1849918612 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:04.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108214800 0x557108214be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb68a59c700 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108215120 0x55710821e920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 -- 172.21.15.90:0/1849918612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108215120 msgr2=0x55710821e920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108215120 0x55710821e920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 -- 172.21.15.90:0/1849918612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108214800 msgr2=0x557108214be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.955+0000 7fb689d9b700 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108214800 0x557108214be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.956+0000 7fb689d9b700 1 -- 172.21.15.90:0/1849918612 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557107fe7430 con 0x55710821ee60 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.956+0000 7fb689d9b700 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fb684008d20 tx=0x7fb6840099e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e0060cd9da56bb4c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb678ff9700 1 -- 172.21.15.90:0/1849918612 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb684004bb0 con 0x55710821ee60 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb678ff9700 1 -- 172.21.15.90:0/1849918612 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb68400e030 con 0x55710821ee60 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb678ff9700 1 -- 172.21.15.90:0/1849918612 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6840075c0 con 0x55710821ee60 2024-04-06T14:06:04.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb67b7fe700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 0x7fb6740a3a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb68a59c700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 0x7fb6740a3a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb67b7fe700 1 -- 172.21.15.90:0/318931829 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb67c014070 con 0x7fb68c107620 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1849918612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 msgr2=0x557108221250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.957+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fb684008d20 tx=0x7fb6840099e0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1849918612 shutdown_connections 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55710821ee60 0x557108221250 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108214800 0x557108214be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1849918612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108215120 0x55710821e920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1849918612 >> 172.21.15.90:0/1849918612 conn(0x557108173720 msgr2=0x55710820fb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb68a59c700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 0x7fb6740a3a00 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb680009fd0 tx=0x7fb680019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.958+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1849918612 shutdown_connections 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.959+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1849918612 wait complete. 2024-04-06T14:06:04.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.959+0000 7fb69cb2bb80 1 -- start start 2024-04-06T14:06:04.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.959+0000 7fb69cb2bb80 1 Processor -- start 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x55710821e890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x55710821bbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710821c310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557108101e00 con 0x557108215120 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557108101aa0 con 0x55710821ee60 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557107fe9720 con 0x557108214800 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb69cb2bb80 10 client.0 _ll_register_callbacks cb 0x55710810b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x55710821e890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710821c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710821c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49428/0 (socket says 172.21.15.90:49428) 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68ad9d700 1 -- 172.21.15.90:0/2740992629 learned_addr learned my addr 172.21.15.90:0/2740992629 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:04.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb689d9b700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x55710821bbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 -- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 msgr2=0x55710821c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710821c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 -- 172.21.15.90:0/2740992629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 msgr2=0x55710821bbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x55710821bbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.960+0000 7fb68a59c700 1 -- 172.21.15.90:0/2740992629 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6840089b0 con 0x557108214800 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x55710821e890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fb680009810 tx=0x7fb680009c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=21e6569c2ae7832a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb671ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68001c040 con 0x557108214800 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb680005450 con 0x557108214800 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557108126400 con 0x557108214800 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb680005610 con 0x557108214800 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb69cb2bb80 10 client.25180 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:04.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.961+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x557107f93be0 con 0x557108214800 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557108126400 con 0x557108214800 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb680024750 con 0x557108214800 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 10 client.25180 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 10 client.25180 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 20 client.25180 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55710822e950 0x557108230d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 4 client.25180 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:04.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55710815e780 con 0x55710822e950 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.962+0000 7fb689d9b700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55710822e950 0x557108230d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.963+0000 7fb671ffb700 10 client.25180 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.963+0000 7fb689d9b700 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55710822e950 0x557108230d70 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=df74639926a0919 server_cookie=2a4558feb496a3b9 in_seq=0 out_seq=0 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.963+0000 7fb671ffb700 1 -- 172.21.15.90:0/2740992629 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55710815e780 con 0x55710822e950 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.963+0000 7fb671ffb700 10 client.25180 handle_command_reply: tid=0 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.963+0000 7fb69cb2bb80 1 client.25180 shutdown 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 msgr2=0x55710821e890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x55710821e890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fb680009810 tx=0x7fb680009c00 comp rx=0 tx=0).stop 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 shutdown_connections 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710821c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55710822e950 0x557108230d70 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x55710821e890 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb671ffb700 0 client.25180 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2740992629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x55710821bbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 >> 172.21.15.90:0/2740992629 conn(0x557108173720 msgr2=0x55710820eca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 shutdown_connections 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.964+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2740992629 wait complete. 2024-04-06T14:06:04.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.965+0000 7fb69cb2bb80 20 client.25180 trim_cache size 0 max 16384 2024-04-06T14:06:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 Processor -- start 2024-04-06T14:06:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 -- start start 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108215120 0x55710820f5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55710821ee60 0x55710820fae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557108226080 con 0x55710821ee60 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557108210050 con 0x557108214800 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.991+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710822e6e0 con 0x557108215120 2024-04-06T14:06:04.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49436/0 (socket says 172.21.15.90:49436) 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 -- 172.21.15.90:0/2128761180 learned_addr learned my addr 172.21.15.90:0/2128761180 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb689d9b700 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108215120 0x55710820f5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 -- 172.21.15.90:0/2128761180 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108215120 msgr2=0x55710820f5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108215120 0x55710820f5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 -- 172.21.15.90:0/2128761180 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55710821ee60 msgr2=0x55710820fae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55710821ee60 0x55710820fae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 -- 172.21.15.90:0/2128761180 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5571082359a0 con 0x557108214800 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb68a59c700 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fb6800098c0 tx=0x7fb68000b360 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fd6f39e0acc8b423 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb6797fa700 1 -- 172.21.15.90:0/2128761180 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68001c040 con 0x557108214800 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb6797fa700 1 -- 172.21.15.90:0/2128761180 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb680005710 con 0x557108214800 2024-04-06T14:06:04.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.992+0000 7fb6797fa700 1 -- 172.21.15.90:0/2128761180 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb680003ce0 con 0x557108214800 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2128761180 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 msgr2=0x557108225ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fb6800098c0 tx=0x7fb68000b360 comp rx=0 tx=0).stop 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2128761180 shutdown_connections 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55710821ee60 0x55710820fae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557108214800 0x557108225ab0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/2128761180 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108215120 0x55710820f5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.993+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2128761180 >> 172.21.15.90:0/2128761180 conn(0x557108173720 msgr2=0x55710821a4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:04.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.994+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2128761180 shutdown_connections 2024-04-06T14:06:04.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.994+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/2128761180 wait complete. 2024-04-06T14:06:04.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.994+0000 7fb69cb2bb80 1 -- start start 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.994+0000 7fb69cb2bb80 1 Processor -- start 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x557108224af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x557108225030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710827cd10 con 0x557108215120 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710827ce80 con 0x55710821ee60 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55710827d040 con 0x557108214800 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb69cb2bb80 10 client.0 _ll_register_callbacks cb 0x55710810b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:04.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49442/0 (socket says 172.21.15.90:49442) 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 -- 172.21.15.90:0/1528374023 learned_addr learned my addr 172.21.15.90:0/1528374023 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb689d9b700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x557108225030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 -- 172.21.15.90:0/1528374023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 msgr2=0x557108224af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68a59c700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x557108224af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x557108224af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 -- 172.21.15.90:0/1528374023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 msgr2=0x557108225030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x557108225030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 -- 172.21.15.90:0/1528374023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557107f93be0 con 0x55710821ee60 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb68ad9d700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fb67c099df0 tx=0x7fb67c099d50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=532d942fde02c1cb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb671ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb67c0997e0 con 0x55710821ee60 2024-04-06T14:06:04.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.995+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb67c09a0f0 con 0x55710821ee60 2024-04-06T14:06:04.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.996+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb68400e030 con 0x55710821ee60 2024-04-06T14:06:04.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.996+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb67c09df20 con 0x55710821ee60 2024-04-06T14:06:04.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.996+0000 7fb69cb2bb80 10 client.25253 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:04.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.996+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55710820bbe0 con 0x55710821ee60 2024-04-06T14:06:04.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.996+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb68400e030 con 0x55710821ee60 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb67c0a20e0 con 0x55710821ee60 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 10 client.25253 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 10 client.25253 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 20 client.25253 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557108284810 0x557108286c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 4 client.25253 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.997+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x557107fb4d70 con 0x557108284810 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.998+0000 7fb68a59c700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557108284810 0x557108286c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.998+0000 7fb68a59c700 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557108284810 0x557108286c30 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cb033334d2240b67 server_cookie=bb294587b8857e7c in_seq=0 out_seq=0 2024-04-06T14:06:05.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.998+0000 7fb671ffb700 10 client.25253 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.999+0000 7fb671ffb700 1 -- 172.21.15.90:0/1528374023 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x557107fb4d70 con 0x557108284810 2024-04-06T14:06:05.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.999+0000 7fb671ffb700 10 client.25253 handle_command_reply: tid=0 2024-04-06T14:06:05.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:04.999+0000 7fb69cb2bb80 1 client.25253 shutdown 2024-04-06T14:06:05.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 msgr2=0x55710827c7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7fb67c099df0 tx=0x7fb67c099d50 comp rx=0 tx=0).stop 2024-04-06T14:06:05.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 shutdown_connections 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55710821ee60 0x55710827c7d0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557108284810 0x557108286c30 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557108214800 0x557108224af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 --2- 172.21.15.90:0/1528374023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557108215120 0x557108225030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.000+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 >> 172.21.15.90:0/1528374023 conn(0x557108173720 msgr2=0x55710821b110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.001+0000 7fb671ffb700 0 client.25253 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.001+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 shutdown_connections 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.001+0000 7fb69cb2bb80 1 -- 172.21.15.90:0/1528374023 wait complete. 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.001+0000 7fb69cb2bb80 20 client.25253 trim_cache size 0 max 16384 2024-04-06T14:06:05.003 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:05.004 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:06:05.004 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "b9f3170b-ffe7-4470-8bcf-2d2a554f0216", 2024-04-06T14:06:05.004 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:06:05.004 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:05.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 msgr2=0x7fb6740a3a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 0x7fb6740a3a00 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb680009fd0 tx=0x7fb680019040 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 msgr2=0x7fb68c1a54a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 0x7fb68c1a54a0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fb67c00e970 tx=0x7fb67c00ec80 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 shutdown_connections 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6740a1540 0x7fb6740a3a00 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb68c107620 0x7fb68c1a54a0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb68c0ffe60 0x7fb68c106ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 --2- 172.21.15.90:0/318931829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb68c104f80 0x7fb68c1070e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 >> 172.21.15.90:0/318931829 conn(0x7fb68c076db0 msgr2=0x7fb68c0fe850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 shutdown_connections 2024-04-06T14:06:05.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.002+0000 7fb690977700 1 -- 172.21.15.90:0/318931829 wait complete. 2024-04-06T14:06:05.024 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:06:05.024 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:06:05.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.240+0000 7f7fe210e700 1 -- 172.21.15.90:0/266505470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 msgr2=0x7f7fdc103560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.240+0000 7f7fe210e700 1 --2- 172.21.15.90:0/266505470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc103560 secure :-1 s=READY pgs=469 cs=0 l=1 rev1=1 crypto rx=0x7f7fd00098f0 tx=0x7f7fd0009c00 comp rx=0 tx=0).stop 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 -- 172.21.15.90:0/266505470 shutdown_connections 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 --2- 172.21.15.90:0/266505470 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 --2- 172.21.15.90:0/266505470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc103560 unknown :-1 s=CLOSED pgs=469 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 --2- 172.21.15.90:0/266505470 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7fdc103b30 0x7f7fdc107b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 -- 172.21.15.90:0/266505470 >> 172.21.15.90:0/266505470 conn(0x7f7fdc0fe9c0 msgr2=0x7f7fdc100de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 -- 172.21.15.90:0/266505470 shutdown_connections 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 -- 172.21.15.90:0/266505470 wait complete. 2024-04-06T14:06:05.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.241+0000 7f7fe210e700 1 Processor -- start 2024-04-06T14:06:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 -- start start 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc1192c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7fdc10f6b0 0x7f7fdc10fb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fdc119b50 con 0x7f7fdc103180 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fdc114d70 con 0x7f7fdc10f6b0 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fe210e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7fdc114ee0 con 0x7f7fdc1082a0 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.242+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc1192c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc1192c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54732/0 (socket says 172.21.15.90:54732) 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/3753934894 learned_addr learned my addr 172.21.15.90:0/3753934894 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 -- 172.21.15.90:0/3753934894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7fdc10f6b0 msgr2=0x7f7fdc10fb60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7fdc10f6b0 0x7f7fdc10fb60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 -- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 msgr2=0x7f7fdc1192c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc1192c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.243+0000 7f7fdaffd700 1 -- 172.21.15.90:0/3753934894 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7fd0009520 con 0x7f7fdc1082a0 2024-04-06T14:06:05.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10f170 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7fdc06a3c0 tx=0x7f7fcc00b7e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fd8ff9700 1 -- 172.21.15.90:0/3753934894 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fcc011830 con 0x7f7fdc1082a0 2024-04-06T14:06:05.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7fdc1151c0 con 0x7f7fdc1082a0 2024-04-06T14:06:05.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fd8ff9700 1 -- 172.21.15.90:0/3753934894 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7fcc011e50 con 0x7f7fdc1082a0 2024-04-06T14:06:05.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fd8ff9700 1 -- 172.21.15.90:0/3753934894 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fcc010e50 con 0x7f7fdc1082a0 2024-04-06T14:06:05.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.244+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7fdc1156d0 con 0x7f7fdc1082a0 2024-04-06T14:06:05.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.248+0000 7f7fd8ff9700 1 -- 172.21.15.90:0/3753934894 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f7fcc0104f0 con 0x7f7fdc1082a0 2024-04-06T14:06:05.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.249+0000 7f7fee2c2b80 1 Processor -- start 2024-04-06T14:06:05.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.249+0000 7f7fd8ff9700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 0x7f7fc40a3950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.249+0000 7f7fd8ff9700 1 -- 172.21.15.90:0/3753934894 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f7fcc0cc8a0 con 0x7f7fdc1082a0 2024-04-06T14:06:05.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.249+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 0x7f7fc40a3950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.249+0000 7f7fee2c2b80 1 -- start start 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7edc8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7edcde0 0x5575e7edf1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7e2a990 con 0x5575e7ed2780 2024-04-06T14:06:05.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7e28350 con 0x5575e7ed30a0 2024-04-06T14:06:05.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7e23c40 con 0x5575e7edcde0 2024-04-06T14:06:05.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54738/0 (socket says 172.21.15.90:54738) 2024-04-06T14:06:05.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1099930904 learned_addr learned my addr 172.21.15.90:0/1099930904 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7edc8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1099930904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7edcde0 msgr2=0x5575e7edf1d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7edcde0 0x5575e7edf1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1099930904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 msgr2=0x5575e7edc8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7edc8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.250+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1099930904 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5575e7ca53b0 con 0x5575e7ed2780 2024-04-06T14:06:05.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.251+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000fbd0 tx=0x7f7fd000fc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b6d0bd92a95bce35 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fc1ffb700 1 -- 172.21.15.90:0/1099930904 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fd001d070 con 0x5575e7ed2780 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fc1ffb700 1 -- 172.21.15.90:0/1099930904 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7fd000f920 con 0x5575e7ed2780 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fc1ffb700 1 -- 172.21.15.90:0/1099930904 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fd0005330 con 0x5575e7ed2780 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1099930904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 msgr2=0x5575e7ed2b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 secure :-1 s=READY pgs=470 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000fbd0 tx=0x7f7fd000fc00 comp rx=0 tx=0).stop 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.252+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 0x7f7fc40a3950 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f7fd400e350 tx=0x7f7fd400b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1099930904 shutdown_connections 2024-04-06T14:06:05.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7edcde0 0x5575e7edf1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ed2b60 unknown :-1 s=CLOSED pgs=470 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1099930904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7edc8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1099930904 >> 172.21.15.90:0/1099930904 conn(0x5575e7e316a0 msgr2=0x5575e7ecda80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1099930904 shutdown_connections 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1099930904 wait complete. 2024-04-06T14:06:05.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.253+0000 7f7fee2c2b80 1 -- start start 2024-04-06T14:06:05.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.255+0000 7f7fee2c2b80 1 Processor -- start 2024-04-06T14:06:05.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 0x5575e7ed9cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7eda210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7dbfd80 con 0x5575e7edcde0 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7dbfa20 con 0x5575e7ed2780 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7ca76a0 con 0x5575e7ed30a0 2024-04-06T14:06:05.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fee2c2b80 10 client.0 _ll_register_callbacks cb 0x5575e7dc9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 0x5575e7ed9cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 0x5575e7ed9cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49458/0 (socket says 172.21.15.90:49458) 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/3812028996 learned_addr learned my addr 172.21.15.90:0/3812028996 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54750/0 (socket says 172.21.15.90:54750) 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.256+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7eda210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 -- 172.21.15.90:0/3812028996 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 msgr2=0x5575e7eda210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7eda210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 -- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 msgr2=0x5575e7ed9cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 0x5575e7ed9cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 -- 172.21.15.90:0/3812028996 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7fdc110690 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f7fd400eaa0 tx=0x7f7fd4004b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb2203937fac5047 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.257+0000 7f7fbaffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fd4015e40 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f7fd4004bb0 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5575e7de4380 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7fd4004d70 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fee2c2b80 10 client.15615 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.258+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5575e7c51b60 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7fd402c030 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7fd4005660 con 0x5575e7edcde0 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 10 client.15615 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 10 client.15615 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:05.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 20 client.15615 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7eecb20 0x5575e7eeef40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 4 client.15615 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5575e7e1c700 con 0x5575e7eecb20 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7eecb20 0x5575e7eeef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.259+0000 7f7fbaffd700 10 client.15615 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.260+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7eecb20 0x5575e7eeef40 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=721827afd74c4627 server_cookie=2535899326e0a6f5 in_seq=0 out_seq=0 2024-04-06T14:06:05.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.261+0000 7f7fbaffd700 1 -- 172.21.15.90:0/3812028996 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5575e7e1c700 con 0x5575e7eecb20 2024-04-06T14:06:05.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.261+0000 7f7fbaffd700 10 client.15615 handle_command_reply: tid=0 2024-04-06T14:06:05.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.261+0000 7f7fee2c2b80 1 client.15615 shutdown 2024-04-06T14:06:05.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 msgr2=0x5575e7eda750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 secure :-1 s=READY pgs=471 cs=0 l=1 rev1=1 crypto rx=0x7f7fd400eaa0 tx=0x7f7fd4004b40 comp rx=0 tx=0).stop 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 shutdown_connections 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7eda750 unknown :-1 s=CLOSED pgs=471 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed2780 0x5575e7ed9cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7eecb20 0x5575e7eeef40 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/3812028996 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7eda210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 >> 172.21.15.90:0/3812028996 conn(0x5575e7e316a0 msgr2=0x5575e7eccc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.263+0000 7f7fbaffd700 0 client.15615 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.265+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 shutdown_connections 2024-04-06T14:06:05.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.265+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/3812028996 wait complete. 2024-04-06T14:06:05.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.265+0000 7f7fee2c2b80 20 client.15615 trim_cache size 0 max 16384 2024-04-06T14:06:05.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.289+0000 7f7fee2c2b80 1 Processor -- start 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 -- start start 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 0x5575e7ee3a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7ecd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7ee3f90 con 0x5575e7edcde0 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7ed8240 con 0x5575e7ed30a0 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7edf1d0 con 0x5575e7ed2780 2024-04-06T14:06:05.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 0x5575e7ee3a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdaffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdaffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:49474/0 (socket says 172.21.15.90:49474) 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdaffd700 1 -- 172.21.15.90:0/1150414207 learned_addr learned my addr 172.21.15.90:0/1150414207 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7ecd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1150414207 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 msgr2=0x5575e7ed1e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1150414207 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 msgr2=0x5575e7ecd3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7ecd3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.290+0000 7f7fdb7fe700 1 -- 172.21.15.90:0/1150414207 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5575e7de9ea0 con 0x5575e7ed2780 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.291+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:06:05.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.291+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 0x5575e7ee3a00 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000fd20 tx=0x7f7fd000bbe0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=947d8c622b231e2f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.291+0000 7f7fc27fc700 1 -- 172.21.15.90:0/1150414207 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fd001d040 con 0x5575e7ed2780 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.291+0000 7f7fc27fc700 1 -- 172.21.15.90:0/1150414207 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7fd000b500 con 0x5575e7ed2780 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.291+0000 7f7fc27fc700 1 -- 172.21.15.90:0/1150414207 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fd00054f0 con 0x5575e7ed2780 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1150414207 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 msgr2=0x5575e7ee3a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 0x5575e7ee3a00 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f7fd000fd20 tx=0x7f7fd000bbe0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1150414207 shutdown_connections 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7edcde0 0x5575e7ecd3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed2780 0x5575e7ee3a00 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/1150414207 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7ed30a0 0x5575e7ed1e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1150414207 >> 172.21.15.90:0/1150414207 conn(0x5575e7e316a0 msgr2=0x5575e7ed1770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1150414207 shutdown_connections 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.292+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/1150414207 wait complete. 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 -- start start 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 Processor -- start 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ee3990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7edcde0 0x5575e7ee2330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7ee2a10 con 0x5575e7ed2780 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7ee2b80 con 0x5575e7edcde0 2024-04-06T14:06:05.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575e7f40920 con 0x5575e7ed30a0 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fee2c2b80 10 client.0 _ll_register_callbacks cb 0x5575e7dc9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fdaffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fdaffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46434/0 (socket says 172.21.15.90:46434) 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.293+0000 7f7fdaffd700 1 -- 172.21.15.90:0/2204967510 learned_addr learned my addr 172.21.15.90:0/2204967510 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdbfff700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7edcde0 0x5575e7ee2330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ee3990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 -- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7edcde0 msgr2=0x5575e7ee2330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7edcde0 0x5575e7ee2330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 -- 172.21.15.90:0/2204967510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 msgr2=0x5575e7ee3990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ee3990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 -- 172.21.15.90:0/2204967510 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7fdc1156d0 con 0x5575e7ed30a0 2024-04-06T14:06:05.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fdaffd700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f7fcc000c00 tx=0x7f7fcc009330 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6d43a55eb32ee123 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:05.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fbaffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:05.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7fcc0b2200 con 0x5575e7ed30a0 2024-04-06T14:06:05.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.294+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f7fcc099890 con 0x5575e7ed30a0 2024-04-06T14:06:05.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7fd000f920 con 0x5575e7ed30a0 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7fcc09d6c0 con 0x5575e7ed30a0 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fee2c2b80 10 client.25198 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5575e7ca53b0 con 0x5575e7ed30a0 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7fd000f920 con 0x5575e7ed30a0 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.295+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7fcc09d170 con 0x5575e7ed30a0 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 10 client.25198 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:05.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 10 client.25198 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 20 client.25198 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7f42080 0x5575e7f444a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 4 client.25198 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5575e7c72cf0 con 0x5575e7f42080 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7f42080 0x5575e7f444a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.296+0000 7f7fdb7fe700 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7f42080 0x5575e7f444a0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1581f158b610a671 server_cookie=f7ba9a6598f66215 in_seq=0 out_seq=0 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fbaffd700 10 client.25198 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fbaffd700 1 -- 172.21.15.90:0/2204967510 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x5575e7c72cf0 con 0x5575e7f42080 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fbaffd700 10 client.25198 handle_command_reply: tid=0 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fee2c2b80 1 client.25198 shutdown 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 msgr2=0x5575e7ee1df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.297+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f7fcc000c00 tx=0x7f7fcc009330 comp rx=0 tx=0).stop 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 shutdown_connections 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5575e7edcde0 0x5575e7ee2330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5575e7ed2780 0x5575e7ee3990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5575e7f42080 0x5575e7f444a0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 --2- 172.21.15.90:0/2204967510 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5575e7ed30a0 0x5575e7ee1df0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 >> 172.21.15.90:0/2204967510 conn(0x5575e7e316a0 msgr2=0x5575e7ed2000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fbaffd700 0 client.25198 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 shutdown_connections 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.298+0000 7f7fee2c2b80 1 -- 172.21.15.90:0/2204967510 wait complete. 2024-04-06T14:06:05.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.299+0000 7f7fee2c2b80 20 client.25198 trim_cache size 0 max 16384 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout: "b9f3170b-ffe7-4470-8bcf-2d2a554f0216": { 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:06:05.306 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "b9f3170b-ffe7-4470-8bcf-2d2a554f0216", 2024-04-06T14:06:05.307 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:06:05.307 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:06:05.307 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:06:05.307 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:05.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 msgr2=0x7f7fc40a3950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 0x7f7fc40a3950 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f7fd400e350 tx=0x7f7fd400b040 comp rx=0 tx=0).stop 2024-04-06T14:06:05.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 msgr2=0x7f7fdc10f170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:05.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10f170 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7fdc06a3c0 tx=0x7f7fcc00b7e0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 shutdown_connections 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7fc40a1490 0x7f7fc40a3950 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7fdc10f6b0 0x7f7fdc10fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7fdc103180 0x7f7fdc1192c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 --2- 172.21.15.90:0/3753934894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7fdc1082a0 0x7f7fdc10f170 secure :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7fdc06a3c0 tx=0x7f7fcc00b7e0 comp rx=0 tx=0).stop 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 >> 172.21.15.90:0/3753934894 conn(0x7f7fdc0fe9c0 msgr2=0x7f7fdc078620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 shutdown_connections 2024-04-06T14:06:05.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:05.300+0000 7f7fe210e700 1 -- 172.21.15.90:0/3753934894 wait complete. 2024-04-06T14:06:05.320 INFO:tasks.cephfs.filesystem:scrub status for tag:b9f3170b-ffe7-4470-8bcf-2d2a554f0216 - {'path': '/', 'tag': 'b9f3170b-ffe7-4470-8bcf-2d2a554f0216', 'options': 'recursive,force'} 2024-04-06T14:06:05.502 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:05 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:06:05.502 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:05 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.502 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:05 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:06:05.502 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:05 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.502 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:05 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:06:05.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:05 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:06:05.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:05 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:05 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:06:05.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:05 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:05 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:06:05.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:05 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:06:05.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:05 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:05 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:06:05.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:05 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:05.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:05 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:06:06.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:06 smithi090 ceph-mon[43420]: pgmap v24: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 115 MiB/s wr, 2.60k op/s 2024-04-06T14:06:06.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:06 smithi134 ceph-mon[47549]: pgmap v24: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 115 MiB/s wr, 2.60k op/s 2024-04-06T14:06:06.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:06 smithi105 ceph-mon[46969]: pgmap v24: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 115 MiB/s wr, 2.60k op/s 2024-04-06T14:06:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 20.0 s, 4037.8 tps, lat 24.884 ms stddev 30.797 2024-04-06T14:06:08.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:08 smithi090 ceph-mon[43420]: pgmap v25: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 52 MiB/s wr, 1.95k op/s 2024-04-06T14:06:08.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:08 smithi134 ceph-mon[47549]: pgmap v25: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 52 MiB/s wr, 1.95k op/s 2024-04-06T14:06:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:08 smithi105 ceph-mon[46969]: pgmap v25: 97 pgs: 97 active+clean; 1.3 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 52 MiB/s wr, 1.95k op/s 2024-04-06T14:06:09.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:09 smithi105 ceph-mon[46969]: pgmap v26: 97 pgs: 97 active+clean; 1.4 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 64 MiB/s wr, 2.21k op/s 2024-04-06T14:06:10.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:09 smithi090 ceph-mon[43420]: pgmap v26: 97 pgs: 97 active+clean; 1.4 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 64 MiB/s wr, 2.21k op/s 2024-04-06T14:06:10.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:09 smithi134 ceph-mon[47549]: pgmap v26: 97 pgs: 97 active+clean; 1.4 GiB data, 1.4 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 64 MiB/s wr, 2.21k op/s 2024-04-06T14:06:11.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:11 smithi105 ceph-mon[46969]: pgmap v27: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 78 MiB/s wr, 2.57k op/s 2024-04-06T14:06:12.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:11 smithi090 ceph-mon[43420]: pgmap v27: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 78 MiB/s wr, 2.57k op/s 2024-04-06T14:06:12.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:11 smithi134 ceph-mon[47549]: pgmap v27: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 78 MiB/s wr, 2.57k op/s 2024-04-06T14:06:13.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 25.0 s, 3572.3 tps, lat 26.077 ms stddev 30.844 2024-04-06T14:06:13.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:13 smithi105 ceph-mon[46969]: pgmap v28: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 58 MiB/s wr, 2.22k op/s 2024-04-06T14:06:14.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:13 smithi090 ceph-mon[43420]: pgmap v28: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 58 MiB/s wr, 2.22k op/s 2024-04-06T14:06:14.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:13 smithi134 ceph-mon[47549]: pgmap v28: 97 pgs: 97 active+clean; 1.5 GiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 58 MiB/s wr, 2.22k op/s 2024-04-06T14:06:15.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:15 smithi105 ceph-mon[46969]: pgmap v29: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 122 KiB/s rd, 96 MiB/s wr, 4.00k op/s 2024-04-06T14:06:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:15 smithi090 ceph-mon[43420]: pgmap v29: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 122 KiB/s rd, 96 MiB/s wr, 4.00k op/s 2024-04-06T14:06:16.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:15 smithi134 ceph-mon[47549]: pgmap v29: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 122 KiB/s rd, 96 MiB/s wr, 4.00k op/s 2024-04-06T14:06:18.028 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 30.0 s, 4532.9 tps, lat 23.567 ms stddev 41.831 2024-04-06T14:06:18.789 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:18 smithi090 ceph-mon[43420]: pgmap v30: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 66 MiB/s wr, 2.42k op/s 2024-04-06T14:06:18.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:18 smithi134 ceph-mon[47549]: pgmap v30: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 66 MiB/s wr, 2.42k op/s 2024-04-06T14:06:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:18 smithi105 ceph-mon[46969]: pgmap v30: 97 pgs: 97 active+clean; 1.8 GiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 66 MiB/s wr, 2.42k op/s 2024-04-06T14:06:19.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:19 smithi105 ceph-mon[46969]: pgmap v31: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 97 KiB/s rd, 73 MiB/s wr, 2.69k op/s 2024-04-06T14:06:20.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:19 smithi090 ceph-mon[43420]: pgmap v31: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 97 KiB/s rd, 73 MiB/s wr, 2.69k op/s 2024-04-06T14:06:20.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:19 smithi134 ceph-mon[47549]: pgmap v31: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 97 KiB/s rd, 73 MiB/s wr, 2.69k op/s 2024-04-06T14:06:21.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:21 smithi105 ceph-mon[46969]: pgmap v32: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 66 MiB/s wr, 2.55k op/s 2024-04-06T14:06:22.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:21 smithi090 ceph-mon[43420]: pgmap v32: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 66 MiB/s wr, 2.55k op/s 2024-04-06T14:06:22.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:21 smithi134 ceph-mon[47549]: pgmap v32: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 66 MiB/s wr, 2.55k op/s 2024-04-06T14:06:23.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 35.0 s, 4800.2 tps, lat 20.868 ms stddev 24.515 2024-04-06T14:06:23.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:23 smithi105 ceph-mon[46969]: pgmap v33: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 50 MiB/s wr, 2.15k op/s 2024-04-06T14:06:24.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:23 smithi090 ceph-mon[43420]: pgmap v33: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 50 MiB/s wr, 2.15k op/s 2024-04-06T14:06:24.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:23 smithi134 ceph-mon[47549]: pgmap v33: 97 pgs: 97 active+clean; 1.9 GiB data, 2.2 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 50 MiB/s wr, 2.15k op/s 2024-04-06T14:06:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:25 smithi105 ceph-mon[46969]: pgmap v34: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 66 MiB/s wr, 2.72k op/s 2024-04-06T14:06:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:25 smithi090 ceph-mon[43420]: pgmap v34: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 66 MiB/s wr, 2.72k op/s 2024-04-06T14:06:26.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:25 smithi134 ceph-mon[47549]: pgmap v34: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 66 MiB/s wr, 2.72k op/s 2024-04-06T14:06:27.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:27 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:06:27.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:27 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:06:27.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:27 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:06:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 40.0 s, 4114.7 tps, lat 24.173 ms stddev 25.981 2024-04-06T14:06:28.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:28 smithi090 ceph-mon[43420]: pgmap v35: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 28 MiB/s wr, 947 op/s 2024-04-06T14:06:28.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:28 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:06:28.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:28 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:06:28.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:28 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:06:28.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:28 smithi134 ceph-mon[47549]: pgmap v35: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 28 MiB/s wr, 947 op/s 2024-04-06T14:06:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:28 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:06:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:28 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:06:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:28 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:06:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:28 smithi105 ceph-mon[46969]: pgmap v35: 97 pgs: 97 active+clean; 2.0 GiB data, 2.3 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 28 MiB/s wr, 947 op/s 2024-04-06T14:06:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:28 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:06:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:28 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:06:28.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:28 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:06:29.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:29 smithi105 ceph-mon[46969]: pgmap v36: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 44 MiB/s wr, 1.47k op/s 2024-04-06T14:06:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:29 smithi090 ceph-mon[43420]: pgmap v36: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 44 MiB/s wr, 1.47k op/s 2024-04-06T14:06:30.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:29 smithi134 ceph-mon[47549]: pgmap v36: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 44 MiB/s wr, 1.47k op/s 2024-04-06T14:06:31.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:31 smithi105 ceph-mon[46969]: pgmap v37: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 37 MiB/s wr, 1.21k op/s 2024-04-06T14:06:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:31 smithi090 ceph-mon[43420]: pgmap v37: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 37 MiB/s wr, 1.21k op/s 2024-04-06T14:06:32.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:31 smithi134 ceph-mon[47549]: pgmap v37: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 37 MiB/s wr, 1.21k op/s 2024-04-06T14:06:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 45.0 s, 2857.6 tps, lat 35.111 ms stddev 97.485 2024-04-06T14:06:33.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:33 smithi105 ceph-mon[46969]: pgmap v38: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 33 MiB/s wr, 1.09k op/s 2024-04-06T14:06:34.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:33 smithi090 ceph-mon[43420]: pgmap v38: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 33 MiB/s wr, 1.09k op/s 2024-04-06T14:06:34.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:33 smithi134 ceph-mon[47549]: pgmap v38: 97 pgs: 97 active+clean; 2.2 GiB data, 2.5 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 33 MiB/s wr, 1.09k op/s 2024-04-06T14:06:35.322 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:06:35.322 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:06:35.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.528+0000 7f46ab6ab700 1 -- 172.21.15.90:0/1604713232 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a410f920 msgr2=0x7f46a41082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.528+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/1604713232 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a410f920 0x7f46a41082f0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f469c00b0a0 tx=0x7f469c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.530+0000 7f46ab6ab700 1 -- 172.21.15.90:0/1604713232 shutdown_connections 2024-04-06T14:06:35.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.530+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/1604713232 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a411a590 0x7f46a411c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.530+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/1604713232 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a410f4b0 0x7f46a4107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.530+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/1604713232 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a410f920 0x7f46a41082f0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.530+0000 7f46ab6ab700 1 -- 172.21.15.90:0/1604713232 >> 172.21.15.90:0/1604713232 conn(0x7f46a406be00 msgr2=0x7f46a406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.535+0000 7f46ab6ab700 1 -- 172.21.15.90:0/1604713232 shutdown_connections 2024-04-06T14:06:35.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.537+0000 7f46ab6ab700 1 -- 172.21.15.90:0/1604713232 wait complete. 2024-04-06T14:06:35.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 Processor -- start 2024-04-06T14:06:35.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 -- start start 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a410f920 0x7f46a4119bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 0x7f46a41146b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a411a440 con 0x7f46a410f920 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a41bb250 con 0x7f46a4114200 2024-04-06T14:06:35.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46ab6ab700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46a41bb410 con 0x7f46a411a590 2024-04-06T14:06:35.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46a8c46700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46a9c48700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 0x7f46a41146b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46a8c46700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45934/0 (socket says 172.21.15.90:45934) 2024-04-06T14:06:35.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.538+0000 7f46a9c48700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 0x7f46a41146b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60550/0 (socket says 172.21.15.90:60550) 2024-04-06T14:06:35.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 -- 172.21.15.90:0/921650888 learned_addr learned my addr 172.21.15.90:0/921650888 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a9447700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a410f920 0x7f46a4119bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 -- 172.21.15.90:0/921650888 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 msgr2=0x7f46a41146b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 0x7f46a41146b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 -- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a410f920 msgr2=0x7f46a4119bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a410f920 0x7f46a4119bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 -- 172.21.15.90:0/921650888 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f469c009d00 con 0x7f46a411a590 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46a8c46700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f469c009610 tx=0x7f469c007b60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f469a7fc700 1 -- 172.21.15.90:0/921650888 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f469c00e040 con 0x7f46a411a590 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f469a7fc700 1 -- 172.21.15.90:0/921650888 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f469c012dc0 con 0x7f46a411a590 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f469a7fc700 1 -- 172.21.15.90:0/921650888 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f469c01b450 con 0x7f46a411a590 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46a41bb690 con 0x7f46a411a590 2024-04-06T14:06:35.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.539+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46a41bbac0 con 0x7f46a411a590 2024-04-06T14:06:35.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 Processor -- start 2024-04-06T14:06:35.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 -- start start 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d791b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d79e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d6e9990 con 0x55d43d791780 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d6e7350 con 0x55d43d7920a0 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46b785fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d6e2c40 con 0x55d43d79bde0 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f469a7fc700 1 -- 172.21.15.90:0/921650888 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f469c01b5b0 con 0x7f46a411a590 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46a9447700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46a9447700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60556/0 (socket says 172.21.15.90:60556) 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46a9447700 1 -- 172.21.15.90:0/4013165347 learned_addr learned my addr 172.21.15.90:0/4013165347 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.544+0000 7f46a8c46700 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d79e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9c48700 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d791b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 -- 172.21.15.90:0/4013165347 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 msgr2=0x55d43d79e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d79e1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 -- 172.21.15.90:0/4013165347 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 msgr2=0x55d43d791b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d791b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 -- 172.21.15.90:0/4013165347 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d43d5643b0 con 0x55d43d7920a0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f46a9447700 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f4694006ab0 tx=0x7f469400bb10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7e40b0d91934351f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f468f7fe700 1 -- 172.21.15.90:0/4013165347 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46940125f0 con 0x55d43d7920a0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f468f7fe700 1 -- 172.21.15.90:0/4013165347 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4694012c10 con 0x55d43d7920a0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f468f7fe700 1 -- 172.21.15.90:0/4013165347 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4694014cc0 con 0x55d43d7920a0 2024-04-06T14:06:35.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.545+0000 7f469a7fc700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 0x7f46900a39d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.546+0000 7f469a7fc700 1 -- 172.21.15.90:0/921650888 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f469c0cedd0 con 0x7f46a411a590 2024-04-06T14:06:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.546+0000 7f46a9c48700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 0x7f46900a39d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.546+0000 7f46b785fb80 1 -- 172.21.15.90:0/4013165347 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 msgr2=0x55d43d79b8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.546+0000 7f46b785fb80 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f4694006ab0 tx=0x7f469400bb10 comp rx=0 tx=0).stop 2024-04-06T14:06:35.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 -- 172.21.15.90:0/4013165347 shutdown_connections 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d79e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d791b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 --2- 172.21.15.90:0/4013165347 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d79b8a0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 -- 172.21.15.90:0/4013165347 >> 172.21.15.90:0/4013165347 conn(0x55d43d6f06a0 msgr2=0x55d43d78ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46a9c48700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 0x7f46900a39d0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f46a406a110 tx=0x7f46a0008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.548+0000 7f46b785fb80 1 -- 172.21.15.90:0/4013165347 shutdown_connections 2024-04-06T14:06:35.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.549+0000 7f46b785fb80 1 -- 172.21.15.90:0/4013165347 wait complete. 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.549+0000 7f46b785fb80 1 -- start start 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.549+0000 7f46b785fb80 1 Processor -- start 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d79b780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d798b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7990a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d67ed80 con 0x55d43d791780 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d67ea20 con 0x55d43d7920a0 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d5666a0 con 0x55d43d79bde0 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46b785fb80 10 client.0 _ll_register_callbacks cb 0x55d43d688470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9447700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d79b780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7990a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9447700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d79b780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34946/0 (socket says 172.21.15.90:34946) 2024-04-06T14:06:35.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9447700 1 -- 172.21.15.90:0/2189265165 learned_addr learned my addr 172.21.15.90:0/2189265165 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a8c46700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d798b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 -- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 msgr2=0x55d43d798b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d798b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 -- 172.21.15.90:0/2189265165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 msgr2=0x55d43d79b780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d79b780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.550+0000 7f46a9c48700 1 -- 172.21.15.90:0/2189265165 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f469400e040 con 0x55d43d79bde0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.551+0000 7f46a9c48700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7990a0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f46a0011540 tx=0x7f46a0004750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fa19d9a505dc2b5f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.551+0000 7f468cff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.551+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46a0015c30 con 0x55d43d79bde0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.551+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f46a0015d90 con 0x55d43d79bde0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.551+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d43d6a3380 con 0x55d43d79bde0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f46a0021440 con 0x55d43d79bde0 2024-04-06T14:06:35.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 10 client.25210 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d43d510b60 con 0x55d43d79bde0 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d43d6a3380 con 0x55d43d79bde0 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f46a002a6f0 con 0x55d43d79bde0 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 10 client.25210 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 10 client.25210 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 20 client.25210 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d7abaa0 0x55d43d7adec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 4 client.25210 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:35.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.552+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d43d6db700 con 0x55d43d7abaa0 2024-04-06T14:06:35.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.553+0000 7f46a9447700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d7abaa0 0x55d43d7adec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.553+0000 7f468cff9700 10 client.25210 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:35.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.553+0000 7f46a9447700 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d7abaa0 0x55d43d7adec0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c9f26d5d6e0dd52a server_cookie=9a9937cf17dbba9b in_seq=0 out_seq=0 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.554+0000 7f468cff9700 1 -- 172.21.15.90:0/2189265165 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d43d6db700 con 0x55d43d7abaa0 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.554+0000 7f468cff9700 10 client.25210 handle_command_reply: tid=0 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.554+0000 7f46b785fb80 1 client.25210 shutdown 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 msgr2=0x55d43d7990a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7990a0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f46a0011540 tx=0x7f46a0004750 comp rx=0 tx=0).stop 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 shutdown_connections 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d7abaa0 0x55d43d7adec0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7990a0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d79b780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f468cff9700 0 client.25210 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:35.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2189265165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d798b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 >> 172.21.15.90:0/2189265165 conn(0x55d43d6f06a0 msgr2=0x55d43d78bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 shutdown_connections 2024-04-06T14:06:35.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.555+0000 7f46b785fb80 1 -- 172.21.15.90:0/2189265165 wait complete. 2024-04-06T14:06:35.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.557+0000 7f46b785fb80 20 client.25210 trim_cache size 0 max 16384 2024-04-06T14:06:35.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 Processor -- start 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 -- start start 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 0x55d43d78c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d7a3000 con 0x55d43d79bde0 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d7996a0 con 0x55d43d7920a0 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46b785fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d794670 con 0x55d43d791780 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a8c46700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a9c48700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 0x55d43d78c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a9447700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a8c46700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60574/0 (socket says 172.21.15.90:60574) 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a9c48700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 0x55d43d78c3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34956/0 (socket says 172.21.15.90:34956) 2024-04-06T14:06:35.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a8c46700 1 -- 172.21.15.90:0/2837781766 learned_addr learned my addr 172.21.15.90:0/2837781766 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.585+0000 7f46a9447700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45952/0 (socket says 172.21.15.90:45952) 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 -- 172.21.15.90:0/2837781766 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 msgr2=0x55d43d7a2ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 -- 172.21.15.90:0/2837781766 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 msgr2=0x55d43d78c3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 0x55d43d78c3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 -- 172.21.15.90:0/2837781766 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d43d6a8ea0 con 0x55d43d791780 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a8c46700 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f46a9447700 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f4694006a40 tx=0x7f46940145b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5d95bb32ac149c56 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f468ffff700 1 -- 172.21.15.90:0/2837781766 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4694024070 con 0x55d43d791780 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f468ffff700 1 -- 172.21.15.90:0/2837781766 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4694014430 con 0x55d43d791780 2024-04-06T14:06:35.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.586+0000 7f468ffff700 1 -- 172.21.15.90:0/2837781766 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46940127c0 con 0x55d43d791780 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- 172.21.15.90:0/2837781766 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 msgr2=0x55d43d7ade90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f4694006a40 tx=0x7f46940145b0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- 172.21.15.90:0/2837781766 shutdown_connections 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d79bde0 0x55d43d78c3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d791780 0x55d43d7ade90 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 --2- 172.21.15.90:0/2837781766 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- 172.21.15.90:0/2837781766 >> 172.21.15.90:0/2837781766 conn(0x55d43d6f06a0 msgr2=0x55d43d7973b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- 172.21.15.90:0/2837781766 shutdown_connections 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- 172.21.15.90:0/2837781766 wait complete. 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.587+0000 7f46b785fb80 1 -- start start 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 Processor -- start 2024-04-06T14:06:35.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d7a2520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7a0b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d7a1270 con 0x55d43d791780 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d7a13e0 con 0x55d43d7920a0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d43d7a15a0 con 0x55d43d79bde0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46b785fb80 10 client.0 _ll_register_callbacks cb 0x55d43d688470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46a9447700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d7a2520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46a9447700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d7a2520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34962/0 (socket says 172.21.15.90:34962) 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46a9447700 1 -- 172.21.15.90:0/753346765 learned_addr learned my addr 172.21.15.90:0/753346765 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46a9c48700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7a0b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.588+0000 7f46a8c46700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 -- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 msgr2=0x55d43d7a2a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 -- 172.21.15.90:0/753346765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 msgr2=0x55d43d7a2520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d7a2520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 -- 172.21.15.90:0/753346765 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46a41bbac0 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46a9c48700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7a0b60 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f46a0000c00 tx=0x7f46a002a620 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2e4d168ad50bc94 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f468cff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46a002a810 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f46a0021650 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.589+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d43d531cf0 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f46a0021810 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 10 client.25216 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d43d5643b0 con 0x55d43d79bde0 2024-04-06T14:06:35.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d43d531cf0 con 0x55d43d79bde0 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f46a00338b0 con 0x55d43d79bde0 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 10 client.25216 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 10 client.25216 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 20 client.25216 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d800c50 0x55d43d803070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 4 client.25216 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:06:35.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.590+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d43d566ce0 con 0x55d43d800c50 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.592+0000 7f46a9447700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d800c50 0x55d43d803070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f468cff9700 10 client.25216 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46a9447700 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d800c50 0x55d43d803070 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e221274ed4b64247 server_cookie=fa07d7844990ec68 in_seq=0 out_seq=0 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f468cff9700 1 -- 172.21.15.90:0/753346765 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55d43d566ce0 con 0x55d43d800c50 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f468cff9700 10 client.25216 handle_command_reply: tid=0 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 client.25216 shutdown 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 msgr2=0x55d43d7a0b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7a0b60 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f46a0000c00 tx=0x7f46a002a620 comp rx=0 tx=0).stop 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 shutdown_connections 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d43d800c50 0x55d43d803070 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d43d79bde0 0x55d43d7a0b60 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d43d791780 0x55d43d7a2520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 --2- 172.21.15.90:0/753346765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d43d7920a0 0x55d43d7a2a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 >> 172.21.15.90:0/753346765 conn(0x55d43d6f06a0 msgr2=0x55d43d7a7210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f468cff9700 0 client.25216 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 shutdown_connections 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.593+0000 7f46b785fb80 1 -- 172.21.15.90:0/753346765 wait complete. 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46b785fb80 20 client.25216 trim_cache size 0 max 16384 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:06:35.597 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:35.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 msgr2=0x7f46900a39d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 0x7f46900a39d0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f46a406a110 tx=0x7f46a0008040 comp rx=0 tx=0).stop 2024-04-06T14:06:35.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 msgr2=0x7f46a4113cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f469c009610 tx=0x7f469c007b60 comp rx=0 tx=0).stop 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 shutdown_connections 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46900a1510 0x7f46900a39d0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46a4114200 0x7f46a41146b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46a410f920 0x7f46a4119bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 --2- 172.21.15.90:0/921650888 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46a411a590 0x7f46a4113cc0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 >> 172.21.15.90:0/921650888 conn(0x7f46a406be00 msgr2=0x7f46a410c080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 shutdown_connections 2024-04-06T14:06:35.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.594+0000 7f46ab6ab700 1 -- 172.21.15.90:0/921650888 wait complete. 2024-04-06T14:06:35.612 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:06:35.613 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:06:35.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 -- 172.21.15.90:0/3539728334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5600ffd60 msgr2=0x7fe560105890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 --2- 172.21.15.90:0/3539728334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5600ffd60 0x7fe560105890 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7fe5540098f0 tx=0x7fe554009c00 comp rx=0 tx=0).stop 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 -- 172.21.15.90:0/3539728334 shutdown_connections 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 --2- 172.21.15.90:0/3539728334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe560105f70 0x7fe56010db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 --2- 172.21.15.90:0/3539728334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe5600ff3b0 0x7fe5600ff790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 --2- 172.21.15.90:0/3539728334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe5600ffd60 0x7fe560105890 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 -- 172.21.15.90:0/3539728334 >> 172.21.15.90:0/3539728334 conn(0x7fe5600760a0 msgr2=0x7fe5600fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 -- 172.21.15.90:0/3539728334 shutdown_connections 2024-04-06T14:06:35.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.802+0000 7fe565fc3700 1 -- 172.21.15.90:0/3539728334 wait complete. 2024-04-06T14:06:35.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 Processor -- start 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 -- start start 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 0x7fe560100320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 0x7fe560100860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe560100da0 0x7fe5601a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5601a5930 con 0x7fe5600ff3b0 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5601a5aa0 con 0x7fe560105f70 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.803+0000 7fe565fc3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5601a5c60 con 0x7fe560100da0 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe560100da0 0x7fe5601a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 0x7fe560100320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 0x7fe560100320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34972/0 (socket says 172.21.15.90:34972) 2024-04-06T14:06:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55f7fe700 1 -- 172.21.15.90:0/60434970 learned_addr learned my addr 172.21.15.90:0/60434970 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55effd700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 0x7fe560100860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55effd700 1 -- 172.21.15.90:0/60434970 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe560100da0 msgr2=0x7fe5601a5360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.804+0000 7fe55effd700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe560100da0 0x7fe5601a5360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55effd700 1 -- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 msgr2=0x7fe560100320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55effd700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 0x7fe560100320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55effd700 1 -- 172.21.15.90:0/60434970 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe550009520 con 0x7fe560105f70 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55effd700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 0x7fe560100860 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fe554009230 tx=0x7fe554010ff0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe554022ce0 con 0x7fe560105f70 2024-04-06T14:06:35.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe554022e40 con 0x7fe560105f70 2024-04-06T14:06:35.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.805+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe554003680 con 0x7fe560105f70 2024-04-06T14:06:35.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.806+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe554009520 con 0x7fe560105f70 2024-04-06T14:06:35.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.806+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe560197120 con 0x7fe560105f70 2024-04-06T14:06:35.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.807+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe560109200 con 0x7fe560105f70 2024-04-06T14:06:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.808+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fe554022300 con 0x7fe560105f70 2024-04-06T14:06:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.809+0000 7fe55cff9700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 0x7fe5480b4b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.809+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe554028030 con 0x7fe560105f70 2024-04-06T14:06:35.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.809+0000 7fe55f7fe700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 0x7fe5480b4b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:35.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.810+0000 7fe55f7fe700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 0x7fe5480b4b10 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fe550009ef0 tx=0x7fe550009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:35.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:35.814+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe554096c30 con 0x7fe560105f70 2024-04-06T14:06:35.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:35 smithi105 ceph-mon[46969]: pgmap v39: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 67 MiB/s wr, 2.82k op/s 2024-04-06T14:06:36.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:35 smithi090 ceph-mon[43420]: pgmap v39: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 67 MiB/s wr, 2.82k op/s 2024-04-06T14:06:36.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:35 smithi134 ceph-mon[47549]: pgmap v39: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 67 MiB/s wr, 2.82k op/s 2024-04-06T14:06:36.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.203+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe560067190 con 0x7fe560105f70 2024-04-06T14:06:36.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.204+0000 7fe55cff9700 1 -- 172.21.15.90:0/60434970 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fe55409a250 con 0x7fe560105f70 2024-04-06T14:06:36.204 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:06:36.205 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:06:36.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 msgr2=0x7fe5480b4b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 0x7fe5480b4b10 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7fe550009ef0 tx=0x7fe550009260 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 msgr2=0x7fe560100860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 0x7fe560100860 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fe554009230 tx=0x7fe554010ff0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 shutdown_connections 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe5480b2650 0x7fe5480b4b10 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe560100da0 0x7fe5601a5360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe5600ff3b0 0x7fe560100320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 --2- 172.21.15.90:0/60434970 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe560105f70 0x7fe560100860 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.208+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 >> 172.21.15.90:0/60434970 conn(0x7fe5600760a0 msgr2=0x7fe5601042a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.209+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 shutdown_connections 2024-04-06T14:06:36.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.209+0000 7fe565fc3700 1 -- 172.21.15.90:0/60434970 wait complete. 2024-04-06T14:06:36.211 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:06:36.229 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:36.229 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 -- 172.21.15.90:0/6856597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe9800ffe70 msgr2=0x7fe980100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 --2- 172.21.15.90:0/6856597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe9800ffe70 0x7fe980100250 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fe970009990 tx=0x7fe970009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 -- 172.21.15.90:0/6856597 shutdown_connections 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 --2- 172.21.15.90:0/6856597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe980104f70 0x7fe980067e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 --2- 172.21.15.90:0/6856597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe9800ffe70 0x7fe980100250 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 --2- 172.21.15.90:0/6856597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980100820 0x7fe980104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 -- 172.21.15.90:0/6856597 >> 172.21.15.90:0/6856597 conn(0x7fe980076d50 msgr2=0x7fe980077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 -- 172.21.15.90:0/6856597 shutdown_connections 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 -- 172.21.15.90:0/6856597 wait complete. 2024-04-06T14:06:36.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.421+0000 7fe985466700 1 Processor -- start 2024-04-06T14:06:36.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 -- start start 2024-04-06T14:06:36.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980104f70 0x7fe98010f310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe98010f850 0x7fe9801a9a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe980112fc0 con 0x7fe980104f70 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe9801a9fe0 con 0x7fe9800ffe70 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe985466700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe9801aa1a0 con 0x7fe98010f850 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.422+0000 7fe97effd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60628/0 (socket says 172.21.15.90:60628) 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 -- 172.21.15.90:0/826563461 learned_addr learned my addr 172.21.15.90:0/826563461 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980104f70 0x7fe98010f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 -- 172.21.15.90:0/826563461 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe98010f850 msgr2=0x7fe9801a9a70 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe98010f850 0x7fe9801a9a70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 -- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980104f70 msgr2=0x7fe98010f310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980104f70 0x7fe98010f310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.423+0000 7fe97effd700 1 -- 172.21.15.90:0/826563461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe9700095f0 con 0x7fe9800ffe70 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.424+0000 7fe97effd700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fe97000f690 tx=0x7fe97000f770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.424+0000 7fe967fff700 1 -- 172.21.15.90:0/826563461 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe97001d070 con 0x7fe9800ffe70 2024-04-06T14:06:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.424+0000 7fe967fff700 1 -- 172.21.15.90:0/826563461 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe970005510 con 0x7fe9800ffe70 2024-04-06T14:06:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.424+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe9801aa420 con 0x7fe9800ffe70 2024-04-06T14:06:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.424+0000 7fe967fff700 1 -- 172.21.15.90:0/826563461 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe970004d40 con 0x7fe9800ffe70 2024-04-06T14:06:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.425+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe9801aa840 con 0x7fe9800ffe70 2024-04-06T14:06:36.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.430+0000 7fe967fff700 1 -- 172.21.15.90:0/826563461 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fe97000fc50 con 0x7fe9800ffe70 2024-04-06T14:06:36.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.431+0000 7fe99161ab80 1 Processor -- start 2024-04-06T14:06:36.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.431+0000 7fe99161ab80 1 -- start start 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258adb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b78a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e25805990 con 0x558e258b7de0 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e25803350 con 0x558e258ad780 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe99161ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e257fec40 con 0x558e258ae0a0 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe97e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe97e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34996/0 (socket says 172.21.15.90:34996) 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe97e7fc700 1 -- 172.21.15.90:0/3590812648 learned_addr learned my addr 172.21.15.90:0/3590812648 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.432+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258adb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe967fff700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 0x7fe9680a3980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 -- 172.21.15.90:0/3590812648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 msgr2=0x558e258b78a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b78a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 -- 172.21.15.90:0/3590812648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 msgr2=0x558e258adb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258adb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 -- 172.21.15.90:0/3590812648 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558e256803b0 con 0x558e258b7de0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe967fff700 1 -- 172.21.15.90:0/826563461 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe9700cdf00 con 0x7fe9800ffe70 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7fe9740077c0 tx=0x7fe974007ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c670cfab96e4261d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 0x7fe9680a3980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe9657fa700 1 -- 172.21.15.90:0/3590812648 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe9740125f0 con 0x558e258b7de0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe9657fa700 1 -- 172.21.15.90:0/3590812648 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe974012c10 con 0x558e258b7de0 2024-04-06T14:06:36.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.434+0000 7fe9657fa700 1 -- 172.21.15.90:0/3590812648 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe974013e40 con 0x558e258b7de0 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.435+0000 7fe99161ab80 1 -- 172.21.15.90:0/3590812648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 msgr2=0x558e258ba1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.435+0000 7fe99161ab80 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 secure :-1 s=READY pgs=472 cs=0 l=1 rev1=1 crypto rx=0x7fe9740077c0 tx=0x7fe974007ad0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 0x7fe9680a3980 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fe978009e60 tx=0x7fe978004ad0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe99161ab80 1 -- 172.21.15.90:0/3590812648 shutdown_connections 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe99161ab80 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258b7de0 0x558e258ba1d0 unknown :-1 s=CLOSED pgs=472 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe99161ab80 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258adb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe99161ab80 1 --2- 172.21.15.90:0/3590812648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b78a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.436+0000 7fe99161ab80 1 -- 172.21.15.90:0/3590812648 >> 172.21.15.90:0/3590812648 conn(0x558e2580c6a0 msgr2=0x558e258a8a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.437+0000 7fe99161ab80 1 -- 172.21.15.90:0/3590812648 shutdown_connections 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.437+0000 7fe99161ab80 1 -- 172.21.15.90:0/3590812648 wait complete. 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.437+0000 7fe99161ab80 1 -- start start 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.437+0000 7fe99161ab80 1 Processor -- start 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 0x558e258b7810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b4ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258b52e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e2579ad80 con 0x558e258ad780 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e2579aa20 con 0x558e258b7de0 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e256826a0 con 0x558e258ae0a0 2024-04-06T14:06:36.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe99161ab80 10 client.0 _ll_register_callbacks cb 0x558e257a4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 0x558e258b7810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 0x558e258b7810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35008/0 (socket says 172.21.15.90:35008) 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97effd700 1 -- 172.21.15.90:0/173720053 learned_addr learned my addr 172.21.15.90:0/173720053 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258b52e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 -- 172.21.15.90:0/173720053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 msgr2=0x558e258b4ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b4ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 -- 172.21.15.90:0/173720053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 msgr2=0x558e258b7810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 0x558e258b7810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.438+0000 7fe97f7fe700 1 -- 172.21.15.90:0/173720053 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe97400e040 con 0x558e258b7de0 2024-04-06T14:06:36.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.439+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258b52e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fe978017470 tx=0x7fe978009540 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2898c0fce6e9ed13 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.439+0000 7fe962ff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.440+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe97801c070 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.440+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe978006e40 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.440+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558e257bf380 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe978011070 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 10 client.25295 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x558e2562cb60 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe97802d020 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe978015cf0 con 0x558e258b7de0 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 10 client.25295 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 10 client.25295 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 20 client.25295 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:36.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e258d8a90 0x558e258daeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 4 client.25295 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:36.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.441+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558e257f7700 con 0x558e258d8a90 2024-04-06T14:06:36.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.442+0000 7fe97effd700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e258d8a90 0x558e258daeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.443+0000 7fe97effd700 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e258d8a90 0x558e258daeb0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=29c234a9f316e546 server_cookie=745e0a3ac26ce5aa in_seq=0 out_seq=0 2024-04-06T14:06:36.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.443+0000 7fe962ff5700 10 client.25295 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:36.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.443+0000 7fe962ff5700 1 -- 172.21.15.90:0/173720053 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x558e257f7700 con 0x558e258d8a90 2024-04-06T14:06:36.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.443+0000 7fe962ff5700 10 client.25295 handle_command_reply: tid=0 2024-04-06T14:06:36.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 client.25295 shutdown 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 msgr2=0x558e258b52e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258b52e0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fe978017470 tx=0x7fe978009540 comp rx=0 tx=0).stop 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 shutdown_connections 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e258d8a90 0x558e258daeb0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258b52e0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ad780 0x558e258b7810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe962ff5700 0 client.25295 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 --2- 172.21.15.90:0/173720053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ae0a0 0x558e258b4ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 >> 172.21.15.90:0/173720053 conn(0x558e2580c6a0 msgr2=0x558e258a7c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 shutdown_connections 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 1 -- 172.21.15.90:0/173720053 wait complete. 2024-04-06T14:06:36.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.444+0000 7fe99161ab80 20 client.25295 trim_cache size 0 max 16384 2024-04-06T14:06:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.462+0000 7fe99161ab80 1 Processor -- start 2024-04-06T14:06:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 -- start start 2024-04-06T14:06:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258bea90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258b4430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258b6140 con 0x558e258ae0a0 2024-04-06T14:06:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258b5b20 con 0x558e258ad780 2024-04-06T14:06:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe99161ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258aea60 con 0x558e258b7de0 2024-04-06T14:06:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46006/0 (socket says 172.21.15.90:46006) 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.463+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2378537020 learned_addr learned my addr 172.21.15.90:0/2378537020 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97effd700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258bea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258b4430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2378537020 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 msgr2=0x558e258bea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258bea90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2378537020 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 msgr2=0x558e258b4430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258b4430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2378537020 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558e257c4ea0 con 0x558e258b7de0 2024-04-06T14:06:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258b4430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fe978000f80 tx=0x7fe978017f80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6286a9316382a878 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe965ffb700 1 -- 172.21.15.90:0/2378537020 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe97801c070 con 0x558e258b7de0 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe965ffb700 1 -- 172.21.15.90:0/2378537020 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe97802d070 con 0x558e258b7de0 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.464+0000 7fe965ffb700 1 -- 172.21.15.90:0/2378537020 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe978017250 con 0x558e258b7de0 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 -- 172.21.15.90:0/2378537020 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 msgr2=0x558e258b4970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fe978000f80 tx=0x7fe978017f80 comp rx=0 tx=0).stop 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 -- 172.21.15.90:0/2378537020 shutdown_connections 2024-04-06T14:06:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258b7de0 0x558e258b4970 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258ad780 0x558e258bea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2378537020 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258b4430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 -- 172.21.15.90:0/2378537020 >> 172.21.15.90:0/2378537020 conn(0x558e2580c6a0 msgr2=0x558e258b3470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 -- 172.21.15.90:0/2378537020 shutdown_connections 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.465+0000 7fe99161ab80 1 -- 172.21.15.90:0/2378537020 wait complete. 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.466+0000 7fe99161ab80 1 -- start start 2024-04-06T14:06:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.466+0000 7fe99161ab80 1 Processor -- start 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ad780 0x558e258be900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258bcdf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258bd870 con 0x558e258ae0a0 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258bd9b0 con 0x558e258b7de0 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558e258bdb70 con 0x558e258ad780 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe99161ab80 10 client.0 _ll_register_callbacks cb 0x558e257a4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe97f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60654/0 (socket says 172.21.15.90:60654) 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2181487675 learned_addr learned my addr 172.21.15.90:0/2181487675 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe97effd700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ad780 0x558e258be900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.467+0000 7fe97e7fc700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258bcdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2181487675 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ad780 msgr2=0x558e258be900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ad780 0x558e258be900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2181487675 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 msgr2=0x558e258bcdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258bcdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 -- 172.21.15.90:0/2181487675 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558e2562cb60 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe97f7fe700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe97800c800 tx=0x7fe978006eb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eebb239cc5556966 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe962ff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe97801c040 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe97802e080 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe974012c10 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.468+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe97800ab30 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.469+0000 7fe99161ab80 10 client.25301 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.469+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x558e25684cd0 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.469+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe974012c10 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe978032430 con 0x558e258b7de0 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 10 client.25301 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 10 client.25301 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:06:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 20 client.25301 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e2593eb10 0x558e25940f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 4 client.25301 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x558e2564dcf0 con 0x558e2593eb10 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.470+0000 7fe97effd700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e2593eb10 0x558e25940f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.471+0000 7fe97effd700 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e2593eb10 0x558e25940f00 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=b9b97b9219dc7b8c server_cookie=4c7c036411967bf5 in_seq=0 out_seq=0 2024-04-06T14:06:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.471+0000 7fe962ff5700 10 client.25301 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.471+0000 7fe962ff5700 1 -- 172.21.15.90:0/2181487675 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x558e2564dcf0 con 0x558e2593eb10 2024-04-06T14:06:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.471+0000 7fe962ff5700 10 client.25301 handle_command_reply: tid=0 2024-04-06T14:06:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.471+0000 7fe99161ab80 1 client.25301 shutdown 2024-04-06T14:06:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.472+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 msgr2=0x558e258bd330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.472+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe97800c800 tx=0x7fe978006eb0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 shutdown_connections 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558e2593eb10 0x558e25940f00 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558e258b7de0 0x558e258bd330 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558e258ad780 0x558e258be900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 --2- 172.21.15.90:0/2181487675 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558e258ae0a0 0x558e258bcdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 >> 172.21.15.90:0/2181487675 conn(0x558e2580c6a0 msgr2=0x558e258ac770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe962ff5700 0 client.25301 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 shutdown_connections 2024-04-06T14:06:36.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.473+0000 7fe99161ab80 1 -- 172.21.15.90:0/2181487675 wait complete. 2024-04-06T14:06:36.475 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.474+0000 7fe99161ab80 20 client.25301 trim_cache size 0 max 16384 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 msgr2=0x7fe9680a3980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 0x7fe9680a3980 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7fe978009e60 tx=0x7fe978004ad0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 msgr2=0x7fe98010edd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fe97000f690 tx=0x7fe97000f770 comp rx=0 tx=0).stop 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 shutdown_connections 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe9680a14c0 0x7fe9680a3980 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe98010f850 0x7fe9801a9a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe9800ffe70 0x7fe98010edd0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 --2- 172.21.15.90:0/826563461 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe980104f70 0x7fe98010f310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.475+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 >> 172.21.15.90:0/826563461 conn(0x7fe980076d50 msgr2=0x7fe98010d360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.476+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 shutdown_connections 2024-04-06T14:06:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.476+0000 7fe985466700 1 -- 172.21.15.90:0/826563461 wait complete. 2024-04-06T14:06:36.494 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:36.495 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:06:36.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.701+0000 7fc00ae94700 1 -- 172.21.15.90:0/790579715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 msgr2=0x7fc004100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.701+0000 7fc00ae94700 1 --2- 172.21.15.90:0/790579715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004100250 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fbff0009990 tx=0x7fbff0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 -- 172.21.15.90:0/790579715 shutdown_connections 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 --2- 172.21.15.90:0/790579715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc004104f70 0x7fc00410caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 --2- 172.21.15.90:0/790579715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004100250 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 --2- 172.21.15.90:0/790579715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc004100820 0x7fc004104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 -- 172.21.15.90:0/790579715 >> 172.21.15.90:0/790579715 conn(0x7fc004076d50 msgr2=0x7fc004077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 -- 172.21.15.90:0/790579715 shutdown_connections 2024-04-06T14:06:36.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.702+0000 7fc00ae94700 1 -- 172.21.15.90:0/790579715 wait complete. 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 Processor -- start 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 -- start start 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004197150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc004104f70 0x7fc004197690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc00419c460 con 0x7fc00419b9e0 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc00419c5d0 con 0x7fc004104f70 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.703+0000 7fc00ae94700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc00419c790 con 0x7fc0040ffe70 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35054/0 (socket says 172.21.15.90:35054) 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 -- 172.21.15.90:0/2978511957 learned_addr learned my addr 172.21.15.90:0/2978511957 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc008c30700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004197150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 -- 172.21.15.90:0/2978511957 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 msgr2=0x7fc004197150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004197150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 -- 172.21.15.90:0/2978511957 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc004104f70 msgr2=0x7fc004197690 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:06:36.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc004104f70 0x7fc004197690 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.704+0000 7fc009431700 1 -- 172.21.15.90:0/2978511957 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbff00095f0 con 0x7fc00419b9e0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.705+0000 7fc009431700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fbffc00ef30 tx=0x7fbffc00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.706+0000 7fc001ffb700 1 -- 172.21.15.90:0/2978511957 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbffc018070 con 0x7fc00419b9e0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.706+0000 7fc001ffb700 1 -- 172.21.15.90:0/2978511957 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbffc00f040 con 0x7fc00419b9e0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.706+0000 7fc001ffb700 1 -- 172.21.15.90:0/2978511957 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbffc00cdb0 con 0x7fc00419b9e0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.706+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc004106ac0 con 0x7fc00419b9e0 2024-04-06T14:06:36.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.706+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc004106f80 con 0x7fc00419b9e0 2024-04-06T14:06:36.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.710+0000 7fc017048b80 1 Processor -- start 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.710+0000 7fc017048b80 1 -- start start 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0b780 0x56519ce0bb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 0x56519ce1e0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 0x56519ce209e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519cd63990 con 0x56519ce0b780 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519cd61350 con 0x56519ce1e5f0 2024-04-06T14:06:36.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc017048b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519cd5cc40 con 0x56519ce0c0a0 2024-04-06T14:06:36.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc003fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 0x56519ce209e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc003fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 0x56519ce209e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60680/0 (socket says 172.21.15.90:60680) 2024-04-06T14:06:36.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.711+0000 7fc003fff700 1 -- 172.21.15.90:0/1523954128 learned_addr learned my addr 172.21.15.90:0/1523954128 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 0x56519ce1e0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 -- 172.21.15.90:0/1523954128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 msgr2=0x56519ce209e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 0x56519ce209e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 -- 172.21.15.90:0/1523954128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0b780 msgr2=0x56519ce0bb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0b780 0x56519ce0bb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 -- 172.21.15.90:0/1523954128 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56519cbde3b0 con 0x56519ce0c0a0 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc001ffb700 1 -- 172.21.15.90:0/2978511957 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7fbffc003a00 con 0x7fc00419b9e0 2024-04-06T14:06:36.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.712+0000 7fc008c30700 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 0x56519ce1e0b0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fbff00059d0 tx=0x7fbff000b650 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d0a1f2ef2e23ec94 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fbfeeffd700 1 -- 172.21.15.90:0/1523954128 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff001d070 con 0x56519ce0c0a0 2024-04-06T14:06:36.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fbfeeffd700 1 -- 172.21.15.90:0/1523954128 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbff001e040 con 0x56519ce0c0a0 2024-04-06T14:06:36.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fbfeeffd700 1 -- 172.21.15.90:0/1523954128 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff0004950 con 0x56519ce0c0a0 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fc001ffb700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 0x7fbff40a3980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fc001ffb700 1 -- 172.21.15.90:0/2978511957 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbffc0ccaf0 con 0x7fc00419b9e0 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.713+0000 7fc003fff700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 0x7fbff40a3980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 -- 172.21.15.90:0/1523954128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 msgr2=0x56519ce1e0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 0x56519ce1e0b0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fbff00059d0 tx=0x7fbff000b650 comp rx=0 tx=0).stop 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 -- 172.21.15.90:0/1523954128 shutdown_connections 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce1e5f0 0x56519ce209e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0b780 0x56519ce0bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 --2- 172.21.15.90:0/1523954128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0c0a0 0x56519ce1e0b0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 -- 172.21.15.90:0/1523954128 >> 172.21.15.90:0/1523954128 conn(0x56519cd6a6a0 msgr2=0x56519ce06a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 -- 172.21.15.90:0/1523954128 shutdown_connections 2024-04-06T14:06:36.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc003fff700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 0x7fbff40a3980 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x56519ce0d1f0 tx=0x7fbff800c2d0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.714+0000 7fc017048b80 1 -- 172.21.15.90:0/1523954128 wait complete. 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.715+0000 7fc017048b80 1 -- start start 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.715+0000 7fc017048b80 1 Processor -- start 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1e040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce1b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce1bb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ccf8d80 con 0x56519ce1e5f0 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ccf8a20 con 0x56519ce0c0a0 2024-04-06T14:06:36.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519cbe06a0 con 0x56519ce0b780 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc017048b80 10 client.0 _ll_register_callbacks cb 0x56519cd02470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc009431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce1bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc009431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce1bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35058/0 (socket says 172.21.15.90:35058) 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc009431700 1 -- 172.21.15.90:0/2977873199 learned_addr learned my addr 172.21.15.90:0/2977873199 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc008c30700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.716+0000 7fc003fff700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce1b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.717+0000 7fc008c30700 1 -- 172.21.15.90:0/2977873199 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 msgr2=0x56519ce1b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.717+0000 7fc008c30700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce1b3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.717+0000 7fc008c30700 1 -- 172.21.15.90:0/2977873199 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 msgr2=0x56519ce1bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.717+0000 7fc008c30700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce1bb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.717+0000 7fc008c30700 1 -- 172.21.15.90:0/2977873199 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc00410d360 con 0x56519ce0b780 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.724+0000 7fc008c30700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1e040 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006010 tx=0x7fbff000be20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8c1fc356c22615c7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.724+0000 7fbfe3fff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff001d030 con 0x56519ce0b780 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbff0004810 con 0x56519ce0b780 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56519cd1d380 con 0x56519ce0b780 2024-04-06T14:06:36.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbff00049d0 con 0x56519ce0b780 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fc017048b80 10 client.25243 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.725+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56519cb8ab60 con 0x56519ce0b780 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.726+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56519cd1d380 con 0x56519ce0b780 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbff000fa50 con 0x56519ce0b780 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 10 client.25243 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 10 client.25243 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 20 client.25243 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce2dce0 0x56519ce300d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 4 client.25243 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:36.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56519cd55700 con 0x56519ce2dce0 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.727+0000 7fc003fff700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce2dce0 0x56519ce300d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.728+0000 7fbfe3fff700 10 client.25243 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.728+0000 7fc003fff700 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce2dce0 0x56519ce300d0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=65fa18ac74ed1836 server_cookie=d4720954c3c09bfa in_seq=0 out_seq=0 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fbfe3fff700 1 -- 172.21.15.90:0/2977873199 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56519cd55700 con 0x56519ce2dce0 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fbfe3fff700 10 client.25243 handle_command_reply: tid=0 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 client.25243 shutdown 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 msgr2=0x56519ce1e040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1e040 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006010 tx=0x7fbff000be20 comp rx=0 tx=0).stop 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 shutdown_connections 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce2dce0 0x56519ce300d0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce1bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fbfe3fff700 0 client.25243 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1e040 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 --2- 172.21.15.90:0/2977873199 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce1b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 >> 172.21.15.90:0/2977873199 conn(0x56519cd6a6a0 msgr2=0x56519ce05c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 shutdown_connections 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.729+0000 7fc017048b80 1 -- 172.21.15.90:0/2977873199 wait complete. 2024-04-06T14:06:36.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.731+0000 7fc017048b80 20 client.25243 trim_cache size 0 max 16384 2024-04-06T14:06:36.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 Processor -- start 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 -- start start 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce25240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce063a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519cdfe330 con 0x56519ce1e5f0 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ce25780 con 0x56519ce0c0a0 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc017048b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ce19920 con 0x56519ce0b780 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc008c30700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc008c30700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46050/0 (socket says 172.21.15.90:46050) 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc008c30700 1 -- 172.21.15.90:0/2721498440 learned_addr learned my addr 172.21.15.90:0/2721498440 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.748+0000 7fc009431700 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce063a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc003fff700 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce25240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 -- 172.21.15.90:0/2721498440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 msgr2=0x56519ce25240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce25240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 -- 172.21.15.90:0/2721498440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 msgr2=0x56519ce063a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce063a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 -- 172.21.15.90:0/2721498440 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56519cd22ea0 con 0x56519ce0b780 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fc008c30700 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fbff000ba80 tx=0x7fbff0005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4105095862a70 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fbfef7fe700 1 -- 172.21.15.90:0/2721498440 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff001d040 con 0x56519ce0b780 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fbfef7fe700 1 -- 172.21.15.90:0/2721498440 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbff000f920 con 0x56519ce0b780 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.749+0000 7fbfef7fe700 1 -- 172.21.15.90:0/2721498440 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff002c9e0 con 0x56519ce0b780 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 -- 172.21.15.90:0/2721498440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 msgr2=0x56519ce1b370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fbff000ba80 tx=0x7fbff0005e80 comp rx=0 tx=0).stop 2024-04-06T14:06:36.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 -- 172.21.15.90:0/2721498440 shutdown_connections 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce1e5f0 0x56519ce063a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce0b780 0x56519ce1b370 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 --2- 172.21.15.90:0/2721498440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0c0a0 0x56519ce25240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 -- 172.21.15.90:0/2721498440 >> 172.21.15.90:0/2721498440 conn(0x56519cd6a6a0 msgr2=0x56519ce1ac60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 -- 172.21.15.90:0/2721498440 shutdown_connections 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.750+0000 7fc017048b80 1 -- 172.21.15.90:0/2721498440 wait complete. 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 -- start start 2024-04-06T14:06:36.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 Processor -- start 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0c0a0 0x56519ce234e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 0x56519ce80990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ce80ed0 con 0x56519ce0c0a0 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ce81010 con 0x56519ce0b780 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56519ce92160 con 0x56519ce1e5f0 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.751+0000 7fc017048b80 10 client.0 _ll_register_callbacks cb 0x56519cd02470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc009431700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 0x56519ce80990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60702/0 (socket says 172.21.15.90:60702) 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc009431700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 0x56519ce80990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46058/0 (socket says 172.21.15.90:46058) 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 -- 172.21.15.90:0/1655618653 learned_addr learned my addr 172.21.15.90:0/1655618653 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc003fff700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0c0a0 0x56519ce234e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 -- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 msgr2=0x56519ce80990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 0x56519ce80990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 -- 172.21.15.90:0/1655618653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0c0a0 msgr2=0x56519ce234e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0c0a0 0x56519ce234e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 -- 172.21.15.90:0/1655618653 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56519cb8ab60 con 0x56519ce0b780 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fc008c30700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006010 tx=0x7fbff000bfb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=76f89d0e1c3d5253 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fbfe3fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff001d040 con 0x56519ce0b780 2024-04-06T14:06:36.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.752+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbff0000e40 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.753+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbff001e040 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.753+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbff002c860 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.753+0000 7fc017048b80 10 client.25316 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.753+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56519cbe2cd0 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.753+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbff001e040 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbff0004c70 con 0x56519ce0b780 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 10 client.25316 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 10 client.25316 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 20 client.25316 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce93400 0x56519ce957f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 4 client.25316 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:36.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.754+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x56519cbabcf0 con 0x56519ce93400 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fc003fff700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce93400 0x56519ce957f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fc003fff700 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce93400 0x56519ce957f0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=9cc826993d9e213c server_cookie=82cec6351f34bc0a in_seq=0 out_seq=0 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fbfe3fff700 10 client.25316 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fbfe3fff700 1 -- 172.21.15.90:0/1655618653 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x56519cbabcf0 con 0x56519ce93400 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fbfe3fff700 10 client.25316 handle_command_reply: tid=0 2024-04-06T14:06:36.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.755+0000 7fc017048b80 1 client.25316 shutdown 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 msgr2=0x56519ce22fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fbff0006010 tx=0x7fbff000bfb0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 shutdown_connections 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56519ce93400 0x56519ce957f0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56519ce1e5f0 0x56519ce80990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fbfe3fff700 0 client.25316 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56519ce0b780 0x56519ce22fa0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 --2- 172.21.15.90:0/1655618653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56519ce0c0a0 0x56519ce234e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 >> 172.21.15.90:0/1655618653 conn(0x56519cd6a6a0 msgr2=0x56519ce0b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 shutdown_connections 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 1 -- 172.21.15.90:0/1655618653 wait complete. 2024-04-06T14:06:36.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.756+0000 7fc017048b80 20 client.25316 trim_cache size 0 max 16384 2024-04-06T14:06:36.764 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:36.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 msgr2=0x7fbff40a3980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 0x7fbff40a3980 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x56519ce0d1f0 tx=0x7fbff800c2d0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 msgr2=0x7fc00419be60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 secure :-1 s=READY pgs=473 cs=0 l=1 rev1=1 crypto rx=0x7fbffc00ef30 tx=0x7fbffc00c3a0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 shutdown_connections 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbff40a14c0 0x7fbff40a3980 secure :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0x56519ce0d1f0 tx=0x7fbff800c2d0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc00419b9e0 0x7fc00419be60 unknown :-1 s=CLOSED pgs=473 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0040ffe70 0x7fc004197150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 --2- 172.21.15.90:0/2978511957 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc004104f70 0x7fc004197690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.757+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 >> 172.21.15.90:0/2978511957 conn(0x7fc004076d50 msgr2=0x7fc0040fe660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.758+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 shutdown_connections 2024-04-06T14:06:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.758+0000 7fc00ae94700 1 -- 172.21.15.90:0/2978511957 wait complete. 2024-04-06T14:06:36.778 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:06:36.779 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:06:36.779 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/60434970' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2447236719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd01031e0 msgr2=0x7f2bd01035c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2447236719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd01031e0 0x7f2bd01035c0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc009990 tx=0x7f2bcc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2447236719 shutdown_connections 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2447236719 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0108300 0x7f2bd010fe80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2447236719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd01031e0 0x7f2bd01035c0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2447236719 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd0103b90 0x7f2bd0107bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.970+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2447236719 >> 172.21.15.90:0/2447236719 conn(0x7f2bd00fea40 msgr2=0x7f2bd0100e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.972+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2447236719 shutdown_connections 2024-04-06T14:06:36.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.972+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2447236719 wait complete. 2024-04-06T14:06:36.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.973+0000 7f2bd7b54700 1 Processor -- start 2024-04-06T14:06:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.973+0000 7f2bd7b54700 1 -- start start 2024-04-06T14:06:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd0108300 0x7f2bd0075790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0075cd0 0x7f2bd00715d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2bd0079430 con 0x7f2bd01031e0 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2bd0071b10 con 0x7f2bd0108300 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd7b54700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2bd0071cd0 con 0x7f2bd0075cd0 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35084/0 (socket says 172.21.15.90:35084) 2024-04-06T14:06:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2605823580 learned_addr learned my addr 172.21.15.90:0/2605823580 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0075cd0 0x7f2bd00715d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2605823580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0075cd0 msgr2=0x7f2bd00715d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0075cd0 0x7f2bd00715d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2605823580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd0108300 msgr2=0x7f2bd0075790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd0108300 0x7f2bd0075790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2605823580 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2bcc0095f0 con 0x7f2bd01031e0 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.974+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc006010 tx=0x7f2bcc003b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.975+0000 7f2bc6ffd700 1 -- 172.21.15.90:0/2605823580 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc01d070 con 0x7f2bd01031e0 2024-04-06T14:06:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.975+0000 7f2bc6ffd700 1 -- 172.21.15.90:0/2605823580 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2bcc005580 con 0x7f2bd01031e0 2024-04-06T14:06:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.975+0000 7f2bc6ffd700 1 -- 172.21.15.90:0/2605823580 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc00fd90 con 0x7f2bd01031e0 2024-04-06T14:06:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.975+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2bd0071ef0 con 0x7f2bd01031e0 2024-04-06T14:06:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.975+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2bd00723b0 con 0x7f2bd01031e0 2024-04-06T14:06:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.978+0000 7f2bc6ffd700 1 -- 172.21.15.90:0/2605823580 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f2bcc00f460 con 0x7f2bd01031e0 2024-04-06T14:06:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.979+0000 7f2bc6ffd700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 0x7f2bbc0a3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.979+0000 7f2bc6ffd700 1 -- 172.21.15.90:0/2605823580 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2bcc0cdfc0 con 0x7f2bd01031e0 2024-04-06T14:06:36.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.980+0000 7f2bd50ef700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 0x7f2bbc0a3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.980+0000 7f2bd50ef700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 0x7f2bbc0a3a50 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f2bd0076a10 tx=0x7f2bc0006d20 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 Processor -- start 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 -- start start 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e20a0 0x5594059eb8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059ee1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559405939990 con 0x5594059ebde0 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559405937350 con 0x5594059e20a0 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559405932c40 con 0x5594059e1780 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2bd60f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059ee1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46074/0 (socket says 172.21.15.90:46074) 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2bd58f0700 1 -- 172.21.15.90:0/235682540 learned_addr learned my addr 172.21.15.90:0/235682540 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.981+0000 7f2bd50ef700 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e20a0 0x5594059eb8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 -- 172.21.15.90:0/235682540 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e20a0 msgr2=0x5594059eb8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e20a0 0x5594059eb8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 -- 172.21.15.90:0/235682540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 msgr2=0x5594059ee1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059ee1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 -- 172.21.15.90:0/235682540 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5594057b43b0 con 0x5594059e1780 2024-04-06T14:06:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc0979a0 tx=0x7f2bcc009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d0ac4c8f7cfef684 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bbbfff700 1 -- 172.21.15.90:0/235682540 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc020e50 con 0x5594059e1780 2024-04-06T14:06:36.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bbbfff700 1 -- 172.21.15.90:0/235682540 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2bcc09b630 con 0x5594059e1780 2024-04-06T14:06:36.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.982+0000 7f2bbbfff700 1 -- 172.21.15.90:0/235682540 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc09b3d0 con 0x5594059e1780 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.983+0000 7f2be3d08b80 1 -- 172.21.15.90:0/235682540 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 msgr2=0x5594059e1b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.983+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc0979a0 tx=0x7f2bcc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 -- 172.21.15.90:0/235682540 shutdown_connections 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059ee1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e1780 0x5594059e1b60 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/235682540 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e20a0 0x5594059eb8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 -- 172.21.15.90:0/235682540 >> 172.21.15.90:0/235682540 conn(0x5594059406a0 msgr2=0x5594059dca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 -- 172.21.15.90:0/235682540 shutdown_connections 2024-04-06T14:06:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 -- 172.21.15.90:0/235682540 wait complete. 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 -- start start 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.984+0000 7f2be3d08b80 1 Processor -- start 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e1780 0x5594059e8c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059e9180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594058ced80 con 0x5594059ebde0 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594058cea20 con 0x5594059e1780 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594057b66a0 con 0x5594059e20a0 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2be3d08b80 10 client.0 _ll_register_callbacks cb 0x5594058d8470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd60f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd50ef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059e9180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd50ef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059e9180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46080/0 (socket says 172.21.15.90:46080) 2024-04-06T14:06:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd60f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35112/0 (socket says 172.21.15.90:35112) 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd50ef700 1 -- 172.21.15.90:0/2182682173 learned_addr learned my addr 172.21.15.90:0/2182682173 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e1780 0x5594059e8c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.985+0000 7f2bd60f1700 1 -- 172.21.15.90:0/2182682173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 msgr2=0x5594059e9180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059e9180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bd60f1700 1 -- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e1780 msgr2=0x5594059e8c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e1780 0x5594059e8c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bd60f1700 1 -- 172.21.15.90:0/2182682173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2bd00723b0 con 0x5594059ebde0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f2bc800ba40 tx=0x7f2bc800bd50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=48b1d4db0354e978 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.986+0000 7f2bb97fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bc8010040 con 0x5594059ebde0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2bc800ace0 con 0x5594059ebde0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5594058f3380 con 0x5594059ebde0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2bc800aea0 con 0x5594059ebde0 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 10 client.15675 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559405760b60 con 0x5594059ebde0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5594058f3380 con 0x5594059ebde0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2bc8015460 con 0x5594059ebde0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 10 client.15675 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 10 client.15675 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 20 client.15675 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5594059fb790 0x5594059fdbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 4 client.15675 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55940592b700 con 0x5594059fb790 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.987+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5594059fb790 0x5594059fdbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.988+0000 7f2bb97fa700 10 client.15675 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.988+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5594059fb790 0x5594059fdbb0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6fe267c732ca4fa server_cookie=460916796521b869 in_seq=0 out_seq=0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.988+0000 7f2bb97fa700 1 -- 172.21.15.90:0/2182682173 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55940592b700 con 0x5594059fb790 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.988+0000 7f2bb97fa700 10 client.15675 handle_command_reply: tid=0 2024-04-06T14:06:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 client.15675 shutdown 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 msgr2=0x5594059e96c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 secure :-1 s=READY pgs=475 cs=0 l=1 rev1=1 crypto rx=0x7f2bc800ba40 tx=0x7f2bc800bd50 comp rx=0 tx=0).stop 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 shutdown_connections 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059ebde0 0x5594059e96c0 unknown :-1 s=CLOSED pgs=475 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059e1780 0x5594059e8c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5594059fb790 0x5594059fdbb0 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2182682173 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059e9180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 >> 172.21.15.90:0/2182682173 conn(0x5594059406a0 msgr2=0x5594059dbc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2bb97fa700 0 client.15675 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 shutdown_connections 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2182682173 wait complete. 2024-04-06T14:06:36.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:36.989+0000 7f2be3d08b80 20 client.15675 trim_cache size 0 max 16384 2024-04-06T14:06:36.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/60434970' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.008+0000 7f2be3d08b80 1 Processor -- start 2024-04-06T14:06:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 -- start start 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059dc3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059ebde0 0x5594059dc8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594059ebb50 con 0x5594059e1780 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594059e3a70 con 0x5594059ebde0 2024-04-06T14:06:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.009+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594059e7190 con 0x5594059e20a0 2024-04-06T14:06:37.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.010+0000 7f2bd50ef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059dc3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.010+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.010+0000 7f2bd58f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35128/0 (socket says 172.21.15.90:35128) 2024-04-06T14:06:37.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.010+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2148655637 learned_addr learned my addr 172.21.15.90:0/2148655637 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:37.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059ebde0 0x5594059dc8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2148655637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 msgr2=0x5594059dc3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059dc3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2148655637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059ebde0 msgr2=0x5594059dc8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059ebde0 0x5594059dc8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 -- 172.21.15.90:0/2148655637 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5594058f8ea0 con 0x5594059e1780 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc09ae70 tx=0x7f2bcc01a040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=38799c100e0409da server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.011+0000 7f2bc4da5700 1 -- 172.21.15.90:0/2148655637 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc097270 con 0x5594059e1780 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2bc4da5700 1 -- 172.21.15.90:0/2148655637 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2bcc09ed60 con 0x5594059e1780 2024-04-06T14:06:37.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2bc4da5700 1 -- 172.21.15.90:0/2148655637 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc09b190 con 0x5594059e1780 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2148655637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 msgr2=0x5594059dddc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 secure :-1 s=READY pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc09ae70 tx=0x7f2bcc01a040 comp rx=0 tx=0).stop 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2148655637 shutdown_connections 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5594059ebde0 0x5594059dc8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059e1780 0x5594059dddc0 secure :-1 s=CLOSED pgs=476 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc09ae70 tx=0x7f2bcc01a040 comp rx=0 tx=0).stop 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/2148655637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059e20a0 0x5594059dc3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.012+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2148655637 >> 172.21.15.90:0/2148655637 conn(0x5594059406a0 msgr2=0x5594059e84d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.013+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2148655637 shutdown_connections 2024-04-06T14:06:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.014+0000 7f2be3d08b80 1 -- 172.21.15.90:0/2148655637 wait complete. 2024-04-06T14:06:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.014+0000 7f2be3d08b80 1 -- start start 2024-04-06T14:06:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.014+0000 7f2be3d08b80 1 Processor -- start 2024-04-06T14:06:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059ebde0 0x5594059f0880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 0x559405a48550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 0x559405a48f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594059f1410 con 0x5594059f0dc0 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559405a4f6f0 con 0x559405a48a90 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559405a4f830 con 0x5594059ebde0 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2be3d08b80 10 client.0 _ll_register_callbacks cb 0x5594058d8470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2bd50ef700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 0x559405a48550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2bd50ef700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 0x559405a48550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35140/0 (socket says 172.21.15.90:35140) 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.015+0000 7f2bd50ef700 1 -- 172.21.15.90:0/1175703331 learned_addr learned my addr 172.21.15.90:0/1175703331 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.016+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 0x559405a48f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.016+0000 7f2bd60f1700 1 -- 172.21.15.90:0/1175703331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059ebde0 msgr2=0x5594059f0880 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.016+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059ebde0 0x5594059f0880 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.016+0000 7f2bd60f1700 1 -- 172.21.15.90:0/1175703331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 msgr2=0x559405a48550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.016+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 0x559405a48550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bd60f1700 1 -- 172.21.15.90:0/1175703331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5594057b43b0 con 0x559405a48a90 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bd60f1700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 0x559405a48f10 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f2bc8021040 tx=0x7f2bc800a730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=33d883ed481963a2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bb97fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2bc8010070 con 0x559405a48a90 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2bcc09b630 con 0x559405a48a90 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f2bc8003cc0 con 0x559405a48a90 2024-04-06T14:06:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2bc8003e80 con 0x559405a48a90 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.017+0000 7f2be3d08b80 10 client.25331 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5594057b8cd0 con 0x559405a48a90 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2bcc09b630 con 0x559405a48a90 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2bc8004670 con 0x559405a48a90 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 10 client.25331 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 10 client.25331 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 20 client.25331 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559405a50eb0 0x559405a532a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 4 client.25331 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x559405781cf0 con 0x559405a50eb0 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.018+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559405a50eb0 0x559405a532a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.019+0000 7f2bb97fa700 10 client.25331 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.019+0000 7f2bd58f0700 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559405a50eb0 0x559405a532a0 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ec3f594cb50ef57 server_cookie=1fa702af9ca738ff in_seq=0 out_seq=0 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2bb97fa700 1 -- 172.21.15.90:0/1175703331 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x559405781cf0 con 0x559405a50eb0 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2bb97fa700 10 client.25331 handle_command_reply: tid=0 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 client.25331 shutdown 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 msgr2=0x559405a48f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 0x559405a48f10 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f2bc8021040 tx=0x7f2bc800a730 comp rx=0 tx=0).stop 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 shutdown_connections 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559405a48a90 0x559405a48f10 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559405a50eb0 0x559405a532a0 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5594059ebde0 0x5594059f0880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 --2- 172.21.15.90:0/1175703331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5594059f0dc0 0x559405a48550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 >> 172.21.15.90:0/1175703331 conn(0x5594059406a0 msgr2=0x5594059e0770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:37.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2bb97fa700 0 client.25331 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:37.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 shutdown_connections 2024-04-06T14:06:37.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.020+0000 7f2be3d08b80 1 -- 172.21.15.90:0/1175703331 wait complete. 2024-04-06T14:06:37.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.021+0000 7f2be3d08b80 20 client.25331 trim_cache size 0 max 16384 2024-04-06T14:06:37.023 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.021+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 msgr2=0x7f2bbc0a3a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.021+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 0x7f2bbc0a3a50 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f2bd0076a10 tx=0x7f2bc0006d20 comp rx=0 tx=0).stop 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.021+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 msgr2=0x7f2bd0075250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.021+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 secure :-1 s=READY pgs=474 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc006010 tx=0x7f2bcc003b60 comp rx=0 tx=0).stop 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 shutdown_connections 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2bbc0a1590 0x7f2bbc0a3a50 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2bd0075cd0 0x7f2bd00715d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2bd01031e0 0x7f2bd0075250 unknown :-1 s=CLOSED pgs=474 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 --2- 172.21.15.90:0/2605823580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2bd0108300 0x7f2bd0075790 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:37.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.022+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 >> 172.21.15.90:0/2605823580 conn(0x7f2bd00fea40 msgr2=0x7f2bd00ffde0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:37.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.023+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 shutdown_connections 2024-04-06T14:06:37.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:37.023+0000 7f2bd7b54700 1 -- 172.21.15.90:0/2605823580 wait complete. 2024-04-06T14:06:37.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/60434970' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:06:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 50.0 s, 4120.7 tps, lat 24.297 ms stddev 28.449 2024-04-06T14:06:38.041 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:06:38.041 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:06:38.041 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:06:38.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 -- 172.21.15.90:0/999894206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70180ffe70 msgr2=0x7f7018100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 --2- 172.21.15.90:0/999894206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70180ffe70 0x7f7018100250 secure :-1 s=READY pgs=477 cs=0 l=1 rev1=1 crypto rx=0x7f700c009990 tx=0x7f700c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 -- 172.21.15.90:0/999894206 shutdown_connections 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 --2- 172.21.15.90:0/999894206 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7018104f80 0x7f701810cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 --2- 172.21.15.90:0/999894206 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70180ffe70 0x7f7018100250 unknown :-1 s=CLOSED pgs=477 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 --2- 172.21.15.90:0/999894206 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018100820 0x7f7018104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 -- 172.21.15.90:0/999894206 >> 172.21.15.90:0/999894206 conn(0x7f7018076d40 msgr2=0x7f7018077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 -- 172.21.15.90:0/999894206 shutdown_connections 2024-04-06T14:06:38.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.223+0000 7f701efef700 1 -- 172.21.15.90:0/999894206 wait complete. 2024-04-06T14:06:38.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.224+0000 7f701efef700 1 Processor -- start 2024-04-06T14:06:38.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.224+0000 7f701efef700 1 -- start start 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018104f80 0x7f70181109b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7018114cd0 0x7f7018115180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7018111110 con 0x7f7018114cd0 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70181156c0 con 0x7f70180ffe70 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701efef700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7018115880 con 0x7f7018104f80 2024-04-06T14:06:38.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60778/0 (socket says 172.21.15.90:60778) 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 -- 172.21.15.90:0/754834269 learned_addr learned my addr 172.21.15.90:0/754834269 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f7017fff700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018104f80 0x7f70181109b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 -- 172.21.15.90:0/754834269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018104f80 msgr2=0x7f70181109b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018104f80 0x7f70181109b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 -- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7018114cd0 msgr2=0x7f7018115180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7018114cd0 0x7f7018115180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.225+0000 7f701cd8b700 1 -- 172.21.15.90:0/754834269 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f700c0095f0 con 0x7f70180ffe70 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f701cd8b700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f700c004730 tx=0x7f700c004760 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f7015ffb700 1 -- 172.21.15.90:0/754834269 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700c01d070 con 0x7f70180ffe70 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f7015ffb700 1 -- 172.21.15.90:0/754834269 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f700c005da0 con 0x7f70180ffe70 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f7015ffb700 1 -- 172.21.15.90:0/754834269 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700c022e50 con 0x7f70180ffe70 2024-04-06T14:06:38.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7018115b00 con 0x7f70180ffe70 2024-04-06T14:06:38.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.226+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70181b3010 con 0x7f70180ffe70 2024-04-06T14:06:38.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.228+0000 7f7015ffb700 1 -- 172.21.15.90:0/754834269 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f700c022470 con 0x7f70180ffe70 2024-04-06T14:06:38.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.229+0000 7f7015ffb700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 0x7f70080a3980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.229+0000 7f7015ffb700 1 -- 172.21.15.90:0/754834269 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f700c0ce170 con 0x7f70180ffe70 2024-04-06T14:06:38.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.229+0000 7f7017fff700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 0x7f70080a3980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.230+0000 7f7017fff700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 0x7f70080a3980 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f7018104b00 tx=0x7f7004009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.230+0000 7f702b1a3b80 1 Processor -- start 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.231+0000 7f702b1a3b80 1 -- start start 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c64ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c65b250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c5a6a10 con 0x55e35c64e800 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c5a43d0 con 0x55e35c64f120 2024-04-06T14:06:38.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c59fcc0 con 0x55e35c658e60 2024-04-06T14:06:38.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60790/0 (socket says 172.21.15.90:60790) 2024-04-06T14:06:38.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 -- 172.21.15.90:0/770002308 learned_addr learned my addr 172.21.15.90:0/770002308 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f701d58c700 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c65b250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 -- 172.21.15.90:0/770002308 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 msgr2=0x55e35c65b250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c65b250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 -- 172.21.15.90:0/770002308 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 msgr2=0x55e35c64ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c64ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 -- 172.21.15.90:0/770002308 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e35c421430 con 0x55e35c64f120 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.232+0000 7f7017fff700 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7004011d20 tx=0x7f7004014000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9aeaae19def00bf7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.233+0000 7f7002ffd700 1 -- 172.21.15.90:0/770002308 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700400eb20 con 0x55e35c64f120 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.233+0000 7f7002ffd700 1 -- 172.21.15.90:0/770002308 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7004021070 con 0x55e35c64f120 2024-04-06T14:06:38.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.233+0000 7f7002ffd700 1 -- 172.21.15.90:0/770002308 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700401d670 con 0x55e35c64f120 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.233+0000 7f702b1a3b80 1 -- 172.21.15.90:0/770002308 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 msgr2=0x55e35c658920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.233+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f7004011d20 tx=0x7f7004014000 comp rx=0 tx=0).stop 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 -- 172.21.15.90:0/770002308 shutdown_connections 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c65b250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c64ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/770002308 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64f120 0x55e35c658920 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 -- 172.21.15.90:0/770002308 >> 172.21.15.90:0/770002308 conn(0x55e35c5ad720 msgr2=0x55e35c649b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 -- 172.21.15.90:0/770002308 shutdown_connections 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 -- 172.21.15.90:0/770002308 wait complete. 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.234+0000 7f702b1a3b80 1 -- start start 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 Processor -- start 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 0x55e35c655cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c656210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c656750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c53be00 con 0x55e35c64f120 2024-04-06T14:06:38.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c53baa0 con 0x55e35c64e800 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c423720 con 0x55e35c658e60 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f702b1a3b80 10 client.0 _ll_register_callbacks cb 0x55e35c5454f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 0x55e35c655cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 0x55e35c655cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60794/0 (socket says 172.21.15.90:60794) 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f701cd8b700 1 -- 172.21.15.90:0/4083197525 learned_addr learned my addr 172.21.15.90:0/4083197525 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f7017fff700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c656210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.235+0000 7f701d58c700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c656750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 -- 172.21.15.90:0/4083197525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 msgr2=0x55e35c656750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c656750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 -- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 msgr2=0x55e35c655cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 0x55e35c655cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 -- 172.21.15.90:0/4083197525 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7004017070 con 0x55e35c64f120 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f7017fff700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c656210 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f700400e970 tx=0x7f700401cdb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f27c17e9981f30dc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f6ff7fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700400edf0 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f700401c410 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e35c560400 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f700401c5d0 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f702b1a3b80 10 client.15687 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.236+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e35c3cdbe0 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e35c560400 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7004003810 con 0x55e35c64f120 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 10 client.15687 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 10 client.15687 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 20 client.15687 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c668ab0 0x55e35c66aed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 4 client.15687 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.237+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e35c598780 con 0x55e35c668ab0 2024-04-06T14:06:38.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.238+0000 7f701cd8b700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c668ab0 0x55e35c66aed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.238+0000 7f701cd8b700 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c668ab0 0x55e35c66aed0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a49cd338556d6c3a server_cookie=fe760b1d75d3262d in_seq=0 out_seq=0 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.238+0000 7f6ff7fff700 10 client.15687 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.239+0000 7f6ff7fff700 1 -- 172.21.15.90:0/4083197525 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55e35c598780 con 0x55e35c668ab0 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.239+0000 7f6ff7fff700 10 client.15687 handle_command_reply: tid=0 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.239+0000 7f702b1a3b80 1 client.15687 shutdown 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.239+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 msgr2=0x55e35c656210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.239+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c656210 secure :-1 s=READY pgs=478 cs=0 l=1 rev1=1 crypto rx=0x7f700400e970 tx=0x7f700401cdb0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 shutdown_connections 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c668ab0 0x55e35c66aed0 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c658e60 0x55e35c656750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c64e800 0x55e35c655cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f6ff7fff700 0 client.15687 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/4083197525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c656210 unknown :-1 s=CLOSED pgs=478 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 >> 172.21.15.90:0/4083197525 conn(0x55e35c5ad720 msgr2=0x55e35c648ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 shutdown_connections 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 1 -- 172.21.15.90:0/4083197525 wait complete. 2024-04-06T14:06:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.240+0000 7f702b1a3b80 20 client.15687 trim_cache size 0 max 16384 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 Processor -- start 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 -- start start 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c654e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64f120 0x55e35c65fae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c658c80 con 0x55e35c64e800 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c660020 con 0x55e35c658e60 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c650060 con 0x55e35c64f120 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f701d58c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f701d58c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60806/0 (socket says 172.21.15.90:60806) 2024-04-06T14:06:38.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.263+0000 7f701d58c700 1 -- 172.21.15.90:0/453007823 learned_addr learned my addr 172.21.15.90:0/453007823 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.264+0000 7f701cd8b700 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c654e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 -- 172.21.15.90:0/453007823 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64f120 msgr2=0x55e35c65fae0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:06:38.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64f120 0x55e35c65fae0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 -- 172.21.15.90:0/453007823 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 msgr2=0x55e35c654e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c654e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 -- 172.21.15.90:0/453007823 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e35c655830 con 0x55e35c658e60 2024-04-06T14:06:38.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701cd8b700 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c654e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:06:38.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f701d58c700 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f70100145c0 tx=0x7f701000b040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f452070c1954fa37 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f70037fe700 1 -- 172.21.15.90:0/453007823 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7010010040 con 0x55e35c658e60 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f70037fe700 1 -- 172.21.15.90:0/453007823 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7010014ec0 con 0x55e35c658e60 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.265+0000 7f70037fe700 1 -- 172.21.15.90:0/453007823 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70100046e0 con 0x55e35c658e60 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 -- 172.21.15.90:0/453007823 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 msgr2=0x55e35c64df00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f70100145c0 tx=0x7f701000b040 comp rx=0 tx=0).stop 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 -- 172.21.15.90:0/453007823 shutdown_connections 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c64df00 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64e800 0x55e35c654e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/453007823 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64f120 0x55e35c65fae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 -- 172.21.15.90:0/453007823 >> 172.21.15.90:0/453007823 conn(0x55e35c5ad720 msgr2=0x55e35c64d7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 -- 172.21.15.90:0/453007823 shutdown_connections 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.266+0000 7f702b1a3b80 1 -- 172.21.15.90:0/453007823 wait complete. 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 -- start start 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 Processor -- start 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c6b61a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c65e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c65ef00 con 0x55e35c64f120 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c65f070 con 0x55e35c658e60 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e35c65f230 con 0x55e35c64e800 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.267+0000 7f702b1a3b80 10 client.0 _ll_register_callbacks cb 0x55e35c5454f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701d58c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c65e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46138/0 (socket says 172.21.15.90:46138) 2024-04-06T14:06:38.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701d58c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c65e7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60808/0 (socket says 172.21.15.90:60808) 2024-04-06T14:06:38.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 -- 172.21.15.90:0/2835272112 learned_addr learned my addr 172.21.15.90:0/2835272112 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f7017fff700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c6b61a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 -- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 msgr2=0x55e35c65e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c65e7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 -- 172.21.15.90:0/2835272112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 msgr2=0x55e35c6b61a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c6b61a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 -- 172.21.15.90:0/2835272112 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70181b3010 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f7017fff700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c6b61a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f701cd8b700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f700c009350 tx=0x7f700c005f10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b999691c2724c34f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f6ff7fff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f700c09b560 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e35c3eed70 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f700c0a34f0 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.268+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f700c0b9350 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 10 client.25273 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f70100142a0 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e35c3eed70 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f700c07d070 con 0x55e35c64e800 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 10 client.25273 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:38.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 10 client.25273 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 20 client.25273 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c6be7f0 0x55e35c6c0be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 4 client.25273 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55e35c423d60 con 0x55e35c6be7f0 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.269+0000 7f7017fff700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c6be7f0 0x55e35c6c0be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.270+0000 7f6ff7fff700 10 client.25273 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.270+0000 7f7017fff700 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c6be7f0 0x55e35c6c0be0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aa0f1c3563d04398 server_cookie=d7d5a161b5593a36 in_seq=0 out_seq=0 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.271+0000 7f6ff7fff700 1 -- 172.21.15.90:0/2835272112 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55e35c423d60 con 0x55e35c6be7f0 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.271+0000 7f6ff7fff700 10 client.25273 handle_command_reply: tid=0 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.271+0000 7f702b1a3b80 1 client.25273 shutdown 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 msgr2=0x55e35c6b5c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f700c009350 tx=0x7f700c005f10 comp rx=0 tx=0).stop 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 shutdown_connections 2024-04-06T14:06:38.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e35c658e60 0x55e35c65e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e35c6be7f0 0x55e35c6c0be0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e35c64e800 0x55e35c6b5c60 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 --2- 172.21.15.90:0/2835272112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e35c64f120 0x55e35c6b61a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 >> 172.21.15.90:0/2835272112 conn(0x55e35c5ad720 msgr2=0x55e35c64e080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f6ff7fff700 0 client.25273 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 shutdown_connections 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.272+0000 7f702b1a3b80 1 -- 172.21.15.90:0/2835272112 wait complete. 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:06:38.275 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "9ce0d83d-13e3-47cc-a70e-33a69a5746ae", 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.273+0000 7f702b1a3b80 20 client.25273 trim_cache size 0 max 16384 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 msgr2=0x7f70080a3980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 0x7f70080a3980 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f7018104b00 tx=0x7f7004009260 comp rx=0 tx=0).stop 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 msgr2=0x7f7018110470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f700c004730 tx=0x7f700c004760 comp rx=0 tx=0).stop 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 shutdown_connections 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70080a14c0 0x7f70080a3980 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7018114cd0 0x7f7018115180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70180ffe70 0x7f7018110470 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 --2- 172.21.15.90:0/754834269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7018104f80 0x7f70181109b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 >> 172.21.15.90:0/754834269 conn(0x7f7018076d40 msgr2=0x7f70180fdfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 shutdown_connections 2024-04-06T14:06:38.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.274+0000 7f701efef700 1 -- 172.21.15.90:0/754834269 wait complete. 2024-04-06T14:06:38.293 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:06:38.293 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:06:38.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 -- 172.21.15.90:0/3614374975 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c10cf30 msgr2=0x7f573c114ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 --2- 172.21.15.90:0/3614374975 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c10cf30 0x7f573c114ab0 secure :-1 s=READY pgs=479 cs=0 l=1 rev1=1 crypto rx=0x7f573400b0a0 tx=0x7f573400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 -- 172.21.15.90:0/3614374975 shutdown_connections 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 --2- 172.21.15.90:0/3614374975 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c10cf30 0x7f573c114ab0 unknown :-1 s=CLOSED pgs=479 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 --2- 172.21.15.90:0/3614374975 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c1088b0 0x7f573c108c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 --2- 172.21.15.90:0/3614374975 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c109260 0x7f573c10c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 -- 172.21.15.90:0/3614374975 >> 172.21.15.90:0/3614374975 conn(0x7f573c06be00 msgr2=0x7f573c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 -- 172.21.15.90:0/3614374975 shutdown_connections 2024-04-06T14:06:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.496+0000 7f574213f700 1 -- 172.21.15.90:0/3614374975 wait complete. 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 Processor -- start 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 -- start start 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1088b0 0x7f573c1a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 0x7f573c1a3bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 0x7f573c19dc60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19e230 con 0x7f573c19d7b0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19e370 con 0x7f573c10cf30 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.497+0000 7f574213f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f573c19e530 con 0x7f573c1088b0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 0x7f573c19dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 0x7f573c19dc60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35218/0 (socket says 172.21.15.90:35218) 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1088b0 0x7f573c1a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573bfff700 1 -- 172.21.15.90:0/492424264 learned_addr learned my addr 172.21.15.90:0/492424264 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 0x7f573c1a3bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 -- 172.21.15.90:0/492424264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1088b0 msgr2=0x7f573c1a3680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1088b0 0x7f573c1a3680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 -- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 msgr2=0x7f573c19dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 0x7f573c19dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 -- 172.21.15.90:0/492424264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5734009d00 con 0x7f573c10cf30 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f573affd700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 0x7f573c1a3bc0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f572c00e9c0 tx=0x7f572c00ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.498+0000 7f5738ff9700 1 -- 172.21.15.90:0/492424264 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f572c00c9c0 con 0x7f573c10cf30 2024-04-06T14:06:38.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.499+0000 7f5738ff9700 1 -- 172.21.15.90:0/492424264 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f572c010430 con 0x7f573c10cf30 2024-04-06T14:06:38.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.499+0000 7f5738ff9700 1 -- 172.21.15.90:0/492424264 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f572c018680 con 0x7f573c10cf30 2024-04-06T14:06:38.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.499+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f573c1aa4f0 con 0x7f573c10cf30 2024-04-06T14:06:38.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.499+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f573c1aa9b0 con 0x7f573c10cf30 2024-04-06T14:06:38.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.503+0000 7f5738ff9700 1 -- 172.21.15.90:0/492424264 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 24) v1 ==== 136157+0+0 (secure 0 0 0) 0x7f572c00cb20 con 0x7f573c10cf30 2024-04-06T14:06:38.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.503+0000 7f574e2f3b80 1 Processor -- start 2024-04-06T14:06:38.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.503+0000 7f574e2f3b80 1 -- start start 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 0x563a6a43eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4539e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a396990 con 0x563a6a43f0a0 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a394350 con 0x563a6a4515f0 2024-04-06T14:06:38.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a38fc40 con 0x563a6a43e780 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f5738ff9700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 0x7f57240a3990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35226/0 (socket says 172.21.15.90:35226) 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 -- 172.21.15.90:0/4179822281 learned_addr learned my addr 172.21.15.90:0/4179822281 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573bfff700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 0x563a6a43eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573affd700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4539e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f5738ff9700 1 -- 172.21.15.90:0/492424264 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f572c014070 con 0x7f573c10cf30 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 -- 172.21.15.90:0/4179822281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 msgr2=0x563a6a43eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 0x563a6a43eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 -- 172.21.15.90:0/4179822281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 msgr2=0x563a6a4539e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4539e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573b7fe700 1 -- 172.21.15.90:0/4179822281 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563a6a2113b0 con 0x563a6a43f0a0 2024-04-06T14:06:38.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573bfff700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 0x563a6a43eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:06:38.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.504+0000 7f573bfff700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 0x7f57240a3990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.505+0000 7f573b7fe700 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f5730009d70 tx=0x7f5730011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f040823a754d5cfc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.505+0000 7f573bfff700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 0x7f57240a3990 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f573400b780 tx=0x7f57340099e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.505+0000 7f5721ffb700 1 -- 172.21.15.90:0/4179822281 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5730004220 con 0x563a6a43f0a0 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.506+0000 7f5721ffb700 1 -- 172.21.15.90:0/4179822281 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5730004380 con 0x563a6a43f0a0 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.506+0000 7f5721ffb700 1 -- 172.21.15.90:0/4179822281 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f57300198d0 con 0x563a6a43f0a0 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 -- 172.21.15.90:0/4179822281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 msgr2=0x563a6a4510b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 secure :-1 s=READY pgs=480 cs=0 l=1 rev1=1 crypto rx=0x7f5730009d70 tx=0x7f5730011720 comp rx=0 tx=0).stop 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 -- 172.21.15.90:0/4179822281 shutdown_connections 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4539e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43e780 0x563a6a43eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/4179822281 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4510b0 unknown :-1 s=CLOSED pgs=480 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 -- 172.21.15.90:0/4179822281 >> 172.21.15.90:0/4179822281 conn(0x563a6a39d6a0 msgr2=0x563a6a439a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 -- 172.21.15.90:0/4179822281 shutdown_connections 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.507+0000 7f574e2f3b80 1 -- 172.21.15.90:0/4179822281 wait complete. 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.508+0000 7f574e2f3b80 1 -- start start 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.508+0000 7f574e2f3b80 1 Processor -- start 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a44e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a4515f0 0x563a6a44eee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a32bd80 con 0x563a6a4515f0 2024-04-06T14:06:38.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a32ba20 con 0x563a6a43e780 2024-04-06T14:06:38.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a2136a0 con 0x563a6a43f0a0 2024-04-06T14:06:38.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f574e2f3b80 10 client.0 _ll_register_callbacks cb 0x563a6a335470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60840/0 (socket says 172.21.15.90:60840) 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 -- 172.21.15.90:0/2809211694 learned_addr learned my addr 172.21.15.90:0/2809211694 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573affd700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a44e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 -- 172.21.15.90:0/2809211694 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 msgr2=0x563a6a44e9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a44e9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.509+0000 7f573b7fe700 1 -- 172.21.15.90:0/2809211694 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a4515f0 msgr2=0x563a6a44eee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f573b7fe700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a4515f0 0x563a6a44eee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f573b7fe700 1 -- 172.21.15.90:0/2809211694 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f57300099a0 con 0x563a6a43e780 2024-04-06T14:06:38.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f573b7fe700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f5730009d40 tx=0x7f5730003960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=236b3c289cc8c4d6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f571affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:38.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5730003b00 con 0x563a6a43e780 2024-04-06T14:06:38.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f5730005250 con 0x563a6a43e780 2024-04-06T14:06:38.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.510+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563a6a350380 con 0x563a6a43e780 2024-04-06T14:06:38.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.513+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5730005410 con 0x563a6a43e780 2024-04-06T14:06:38.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.513+0000 7f574e2f3b80 10 client.25361 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:38.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.513+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563a6a1bdb60 con 0x563a6a43e780 2024-04-06T14:06:38.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.513+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563a6a350380 con 0x563a6a43e780 2024-04-06T14:06:38.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5730021040 con 0x563a6a43e780 2024-04-06T14:06:38.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 10 client.25361 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 10 client.25361 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 20 client.25361 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a442f60 0x563a6a445350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 4 client.25361 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.514+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563a6a388700 con 0x563a6a442f60 2024-04-06T14:06:38.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.515+0000 7f573affd700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a442f60 0x563a6a445350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.516+0000 7f571affd700 10 client.25361 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.516+0000 7f573affd700 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a442f60 0x563a6a445350 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c947cafe6ba44805 server_cookie=603df43c05e4f84f in_seq=0 out_seq=0 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.516+0000 7f571affd700 1 -- 172.21.15.90:0/2809211694 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563a6a388700 con 0x563a6a442f60 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.516+0000 7f571affd700 10 client.25361 handle_command_reply: tid=0 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.516+0000 7f574e2f3b80 1 client.25361 shutdown 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 msgr2=0x563a6a44e460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f5730009d40 tx=0x7f5730003960 comp rx=0 tx=0).stop 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 shutdown_connections 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a4515f0 0x563a6a44eee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a44e460 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a442f60 0x563a6a445350 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/2809211694 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a44e9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 >> 172.21.15.90:0/2809211694 conn(0x563a6a39d6a0 msgr2=0x563a6a438c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f571affd700 0 client.25361 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 shutdown_connections 2024-04-06T14:06:38.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.517+0000 7f574e2f3b80 1 -- 172.21.15.90:0/2809211694 wait complete. 2024-04-06T14:06:38.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.518+0000 7f574e2f3b80 20 client.25361 trim_cache size 0 max 16384 2024-04-06T14:06:38.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.539+0000 7f574e2f3b80 1 Processor -- start 2024-04-06T14:06:38.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.539+0000 7f574e2f3b80 1 -- start start 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43e780 0x563a6a448c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4398e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a4401b0 con 0x563a6a43e780 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a440630 con 0x563a6a4515f0 2024-04-06T14:06:38.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a4313c0 con 0x563a6a43f0a0 2024-04-06T14:06:38.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f573affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f573affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46186/0 (socket says 172.21.15.90:46186) 2024-04-06T14:06:38.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f573affd700 1 -- 172.21.15.90:0/1915789762 learned_addr learned my addr 172.21.15.90:0/1915789762 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f573bfff700 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4398e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.541+0000 7f573b7fe700 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43e780 0x563a6a448c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 -- 172.21.15.90:0/1915789762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 msgr2=0x563a6a4398e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4398e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 -- 172.21.15.90:0/1915789762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43e780 msgr2=0x563a6a448c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43e780 0x563a6a448c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 -- 172.21.15.90:0/1915789762 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563a6a355ea0 con 0x563a6a43f0a0 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f573affd700 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f572c00f010 tx=0x7f572c06bf90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=14a4c78ac9be2602 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f57227fc700 1 -- 172.21.15.90:0/1915789762 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f572c069ea0 con 0x563a6a43f0a0 2024-04-06T14:06:38.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f57227fc700 1 -- 172.21.15.90:0/1915789762 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f572c065850 con 0x563a6a43f0a0 2024-04-06T14:06:38.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.542+0000 7f57227fc700 1 -- 172.21.15.90:0/1915789762 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f572c0a2330 con 0x563a6a43f0a0 2024-04-06T14:06:38.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.543+0000 7f574e2f3b80 1 -- 172.21.15.90:0/1915789762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 msgr2=0x563a6a4393a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.543+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f572c00f010 tx=0x7f572c06bf90 comp rx=0 tx=0).stop 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 -- 172.21.15.90:0/1915789762 shutdown_connections 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a4515f0 0x563a6a4398e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43e780 0x563a6a448c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/1915789762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a43f0a0 0x563a6a4393a0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 -- 172.21.15.90:0/1915789762 >> 172.21.15.90:0/1915789762 conn(0x563a6a39d6a0 msgr2=0x563a6a43e000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 -- 172.21.15.90:0/1915789762 shutdown_connections 2024-04-06T14:06:38.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 -- 172.21.15.90:0/1915789762 wait complete. 2024-04-06T14:06:38.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.545+0000 7f574e2f3b80 1 -- start start 2024-04-06T14:06:38.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 Processor -- start 2024-04-06T14:06:38.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4a6520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a4515f0 0x563a6a461c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a4621f0 con 0x563a6a43f0a0 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a462360 con 0x563a6a43e780 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a6a462520 con 0x563a6a4515f0 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f574e2f3b80 10 client.0 _ll_register_callbacks cb 0x563a6a335470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f573b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60852/0 (socket says 172.21.15.90:60852) 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.546+0000 7f573b7fe700 1 -- 172.21.15.90:0/502267451 learned_addr learned my addr 172.21.15.90:0/502267451 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:06:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573bfff700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a4515f0 0x563a6a461c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573b7fe700 1 -- 172.21.15.90:0/502267451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a4515f0 msgr2=0x563a6a461c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573b7fe700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a4515f0 0x563a6a461c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573b7fe700 1 -- 172.21.15.90:0/502267451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 msgr2=0x563a6a4a6520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573b7fe700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4a6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.547+0000 7f573b7fe700 1 -- 172.21.15.90:0/502267451 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f573c1aa9b0 con 0x563a6a43e780 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.548+0000 7f573b7fe700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f5730011740 tx=0x7f5730003cb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=37bcb2242bb4e886 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.548+0000 7f571affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.548+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5730005690 con 0x563a6a43e780 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.548+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f57300357e0 con 0x563a6a43e780 2024-04-06T14:06:38.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.550+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5730004380 con 0x563a6a43e780 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.558+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f57300359a0 con 0x563a6a43e780 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 10 client.25367 fetch_fsmap learned FSMap version 24 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563a6a215cd0 con 0x563a6a43e780 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5730004380 con 0x563a6a43e780 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5730011840 con 0x563a6a43e780 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 10 client.25367 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 10 client.25367 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 20 client.25367 populate_metadata read hostname 'smithi090' 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a4adae0 0x563a6a4aff00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 4 client.25367 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:06:38.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.559+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x563a6a1decf0 con 0x563a6a4adae0 2024-04-06T14:06:38.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.564+0000 7f573affd700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a4adae0 0x563a6a4aff00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "9ce0d83d-13e3-47cc-a70e-33a69a5746ae": { 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "9ce0d83d-13e3-47cc-a70e-33a69a5746ae", 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:06:38.567 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.564+0000 7f573affd700 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a4adae0 0x563a6a4aff00 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7a10945a44b6f71d server_cookie=60418b355d30490 in_seq=0 out_seq=0 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f571affd700 10 client.25367 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f571affd700 1 -- 172.21.15.90:0/502267451 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x563a6a1decf0 con 0x563a6a4adae0 2024-04-06T14:06:38.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f571affd700 10 client.25367 handle_command_reply: tid=0 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 client.25367 shutdown 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 msgr2=0x563a6a4a5fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f5730011740 tx=0x7f5730003cb0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 shutdown_connections 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563a6a4adae0 0x563a6a4aff00 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a6a4515f0 0x563a6a461c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a6a43e780 0x563a6a4a5fe0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 --2- 172.21.15.90:0/502267451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a6a43f0a0 0x563a6a4a6520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 >> 172.21.15.90:0/502267451 conn(0x563a6a39d6a0 msgr2=0x563a6a45c280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f571affd700 0 client.25367 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:06:38.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.565+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 shutdown_connections 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574e2f3b80 1 -- 172.21.15.90:0/502267451 wait complete. 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574e2f3b80 20 client.25367 trim_cache size 0 max 16384 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 msgr2=0x7f57240a3990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 0x7f57240a3990 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f573400b780 tx=0x7f57340099e0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 msgr2=0x7f573c1a3bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 0x7f573c1a3bc0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f572c00e9c0 tx=0x7f572c00ecd0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 shutdown_connections 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f57240a14d0 0x7f57240a3990 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f573c19d7b0 0x7f573c19dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f573c1088b0 0x7f573c1a3680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 --2- 172.21.15.90:0/492424264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f573c10cf30 0x7f573c1a3bc0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:06:38.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.566+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 >> 172.21.15.90:0/492424264 conn(0x7f573c06be00 msgr2=0x7f573c070bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:06:38.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.569+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 shutdown_connections 2024-04-06T14:06:38.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:06:38.569+0000 7f574213f700 1 -- 172.21.15.90:0/492424264 wait complete. 2024-04-06T14:06:38.584 INFO:tasks.cephfs.filesystem:scrub status for tag:9ce0d83d-13e3-47cc-a70e-33a69a5746ae - {'path': '/', 'tag': '9ce0d83d-13e3-47cc-a70e-33a69a5746ae', 'options': 'recursive,force'} 2024-04-06T14:06:38.584 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:38 smithi090 ceph-mon[43420]: pgmap v40: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 51 MiB/s wr, 2.25k op/s 2024-04-06T14:06:38.584 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:38 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:06:38.584 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:38 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:38.585 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:38 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:06:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:38 smithi134 ceph-mon[47549]: pgmap v40: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 51 MiB/s wr, 2.25k op/s 2024-04-06T14:06:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:38 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:06:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:38 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:38 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:06:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:38 smithi105 ceph-mon[46969]: pgmap v40: 97 pgs: 97 active+clean; 2.4 GiB data, 2.6 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 51 MiB/s wr, 2.25k op/s 2024-04-06T14:06:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:38 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:06:38.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:38 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:38.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:38 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:06:39.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:39 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:39.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:39 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:40.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:39 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:06:40.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:40 smithi134 ceph-mon[47549]: pgmap v41: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 60 MiB/s wr, 2.62k op/s 2024-04-06T14:06:40.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:40 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:06:40.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:40 smithi105 ceph-mon[46969]: pgmap v41: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 60 MiB/s wr, 2.62k op/s 2024-04-06T14:06:40.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:40 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:06:41.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:40 smithi090 ceph-mon[43420]: pgmap v41: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 60 MiB/s wr, 2.62k op/s 2024-04-06T14:06:41.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:40 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:06:41.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:41 smithi105 ceph-mon[46969]: pgmap v42: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:42.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:41 smithi090 ceph-mon[43420]: pgmap v42: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:42.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:41 smithi134 ceph-mon[47549]: pgmap v42: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 55.0 s, 4752.4 tps, lat 20.974 ms stddev 22.852 2024-04-06T14:06:44.000 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:43 smithi105 ceph-mon[46969]: pgmap v43: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:44.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:43 smithi090 ceph-mon[43420]: pgmap v43: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:44.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:43 smithi134 ceph-mon[47549]: pgmap v43: 97 pgs: 97 active+clean; 2.4 GiB data, 2.7 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 44 MiB/s wr, 2.10k op/s 2024-04-06T14:06:45.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:45 smithi105 ceph-mon[46969]: pgmap v44: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 53 MiB/s wr, 2.47k op/s 2024-04-06T14:06:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:45 smithi090 ceph-mon[43420]: pgmap v44: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 53 MiB/s wr, 2.47k op/s 2024-04-06T14:06:46.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:45 smithi134 ceph-mon[47549]: pgmap v44: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 53 MiB/s wr, 2.47k op/s 2024-04-06T14:06:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 60.0 s, 4083.0 tps, lat 24.539 ms stddev 26.727 2024-04-06T14:06:48.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:48 smithi134 ceph-mon[47549]: pgmap v45: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 19 MiB/s wr, 738 op/s 2024-04-06T14:06:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:48 smithi105 ceph-mon[46969]: pgmap v45: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 19 MiB/s wr, 738 op/s 2024-04-06T14:06:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:48 smithi090 ceph-mon[43420]: pgmap v45: 97 pgs: 97 active+clean; 2.5 GiB data, 2.8 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 19 MiB/s wr, 738 op/s 2024-04-06T14:06:49.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:49 smithi105 ceph-mon[46969]: pgmap v46: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 214 KiB/s rd, 35 MiB/s wr, 1.36k op/s 2024-04-06T14:06:50.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:49 smithi090 ceph-mon[43420]: pgmap v46: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 214 KiB/s rd, 35 MiB/s wr, 1.36k op/s 2024-04-06T14:06:50.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:49 smithi134 ceph-mon[47549]: pgmap v46: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 214 KiB/s rd, 35 MiB/s wr, 1.36k op/s 2024-04-06T14:06:51.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:51 smithi105 ceph-mon[46969]: pgmap v47: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:52.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:51 smithi090 ceph-mon[43420]: pgmap v47: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:52.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:51 smithi134 ceph-mon[47549]: pgmap v47: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 65.0 s, 4064.6 tps, lat 24.540 ms stddev 27.527 2024-04-06T14:06:53.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:53 smithi105 ceph-mon[46969]: pgmap v48: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:54.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:53 smithi090 ceph-mon[43420]: pgmap v48: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:54.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:53 smithi134 ceph-mon[47549]: pgmap v48: 97 pgs: 97 active+clean; 2.6 GiB data, 3.0 GiB used, 1.0 TiB / 1.0 TiB avail; 209 KiB/s rd, 26 MiB/s wr, 986 op/s 2024-04-06T14:06:55.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:55 smithi105 ceph-mon[46969]: mgrmap e25: x(active, since 91s), standbys: z, y 2024-04-06T14:06:55.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:55 smithi105 ceph-mon[46969]: pgmap v49: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 218 KiB/s rd, 61 MiB/s wr, 2.57k op/s 2024-04-06T14:06:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:55 smithi090 ceph-mon[43420]: mgrmap e25: x(active, since 91s), standbys: z, y 2024-04-06T14:06:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:55 smithi090 ceph-mon[43420]: pgmap v49: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 218 KiB/s rd, 61 MiB/s wr, 2.57k op/s 2024-04-06T14:06:56.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:55 smithi134 ceph-mon[47549]: mgrmap e25: x(active, since 91s), standbys: z, y 2024-04-06T14:06:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:55 smithi134 ceph-mon[47549]: pgmap v49: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 218 KiB/s rd, 61 MiB/s wr, 2.57k op/s 2024-04-06T14:06:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 70.0 s, 3170.4 tps, lat 31.581 ms stddev 66.988 2024-04-06T14:06:58.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:58 smithi134 ceph-mon[47549]: pgmap v50: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 51 MiB/s wr, 2.20k op/s 2024-04-06T14:06:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:58 smithi105 ceph-mon[46969]: pgmap v50: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 51 MiB/s wr, 2.20k op/s 2024-04-06T14:06:59.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:58 smithi090 ceph-mon[43420]: pgmap v50: 97 pgs: 97 active+clean; 2.7 GiB data, 3.1 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 51 MiB/s wr, 2.20k op/s 2024-04-06T14:06:59.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:06:59 smithi105 ceph-mon[46969]: pgmap v51: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 199 KiB/s rd, 63 MiB/s wr, 2.69k op/s 2024-04-06T14:07:00.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:06:59 smithi090 ceph-mon[43420]: pgmap v51: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 199 KiB/s rd, 63 MiB/s wr, 2.69k op/s 2024-04-06T14:07:00.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:06:59 smithi134 ceph-mon[47549]: pgmap v51: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 199 KiB/s rd, 63 MiB/s wr, 2.69k op/s 2024-04-06T14:07:01.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:01 smithi105 ceph-mon[46969]: pgmap v52: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:02.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:01 smithi090 ceph-mon[43420]: pgmap v52: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:02.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:01 smithi134 ceph-mon[47549]: pgmap v52: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 75.0 s, 3273.2 tps, lat 30.580 ms stddev 53.810 2024-04-06T14:07:03.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:03 smithi105 ceph-mon[46969]: pgmap v53: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:04.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:03 smithi090 ceph-mon[43420]: pgmap v53: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:04.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:03 smithi134 ceph-mon[47549]: pgmap v53: 97 pgs: 97 active+clean; 2.7 GiB data, 3.2 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 47 MiB/s wr, 2.07k op/s 2024-04-06T14:07:05.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:05 smithi105 ceph-mon[46969]: pgmap v54: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 65 MiB/s wr, 3.45k op/s 2024-04-06T14:07:06.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:05 smithi090 ceph-mon[43420]: pgmap v54: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 65 MiB/s wr, 3.45k op/s 2024-04-06T14:07:06.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:05 smithi134 ceph-mon[47549]: pgmap v54: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 65 MiB/s wr, 3.45k op/s 2024-04-06T14:07:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 80.0 s, 4433.4 tps, lat 22.573 ms stddev 24.764 2024-04-06T14:07:08.585 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:07:08.585 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:07:08.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 -- 172.21.15.90:0/1666214755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 msgr2=0x7f42581150f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 --2- 172.21.15.90:0/1666214755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f42581150f0 secure :-1 s=READY pgs=481 cs=0 l=1 rev1=1 crypto rx=0x7f4248009990 tx=0x7f4248009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 -- 172.21.15.90:0/1666214755 shutdown_connections 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 --2- 172.21.15.90:0/1666214755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f42581150f0 unknown :-1 s=CLOSED pgs=481 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 --2- 172.21.15.90:0/1666214755 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f4258109b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 --2- 172.21.15.90:0/1666214755 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 -- 172.21.15.90:0/1666214755 >> 172.21.15.90:0/1666214755 conn(0x7f425806be00 msgr2=0x7f425806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 -- 172.21.15.90:0/1666214755 shutdown_connections 2024-04-06T14:07:08.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.782+0000 7f425e946700 1 -- 172.21.15.90:0/1666214755 wait complete. 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.783+0000 7f425e946700 1 Processor -- start 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.783+0000 7f425e946700 1 -- start start 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f4258113c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4258115180 con 0x7f4258112d00 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42581b23a0 con 0x7f425810a070 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425e946700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42581b2560 con 0x7f4258109750 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f4257fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:41912/0 (socket says 172.21.15.90:41912) 2024-04-06T14:07:08.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f4257fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:50786/0 (socket says 172.21.15.90:50786) 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 -- 172.21.15.90:0/1632390256 learned_addr learned my addr 172.21.15.90:0/1632390256 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f425cee3700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f4258113c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 -- 172.21.15.90:0/1632390256 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 msgr2=0x7f42581131b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 -- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 msgr2=0x7f4258113c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f4258113c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f42577fe700 1 -- 172.21.15.90:0/1632390256 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42480095f0 con 0x7f425810a070 2024-04-06T14:07:08.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.784+0000 7f4257fff700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.785+0000 7f42577fe700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f424c00e9a0 tx=0x7f424c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.785+0000 7f42557fa700 1 -- 172.21.15.90:0/1632390256 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f424c009790 con 0x7f425810a070 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.785+0000 7f42557fa700 1 -- 172.21.15.90:0/1632390256 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f424c010430 con 0x7f425810a070 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.785+0000 7f42557fa700 1 -- 172.21.15.90:0/1632390256 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f424c018680 con 0x7f425810a070 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.787+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42581b2840 con 0x7f425810a070 2024-04-06T14:07:08.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.787+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42581b2cd0 con 0x7f425810a070 2024-04-06T14:07:08.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.789+0000 7f42557fa700 1 -- 172.21.15.90:0/1632390256 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f424c0105a0 con 0x7f425810a070 2024-04-06T14:07:08.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.790+0000 7f42557fa700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 0x7f42400a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.790+0000 7f42557fa700 1 -- 172.21.15.90:0/1632390256 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f424c014070 con 0x7f425810a070 2024-04-06T14:07:08.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.790+0000 7f4257fff700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 0x7f42400a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 Processor -- start 2024-04-06T14:07:08.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 -- start start 2024-04-06T14:07:08.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d900780 0x56001d900b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d91e1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d858990 con 0x56001d900780 2024-04-06T14:07:08.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d856350 con 0x56001d9010a0 2024-04-06T14:07:08.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f426aafab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d851c40 con 0x56001d91bdf0 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f42577fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f42577fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:41924/0 (socket says 172.21.15.90:41924) 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.792+0000 7f42577fe700 1 -- 172.21.15.90:0/1055580777 learned_addr learned my addr 172.21.15.90:0/1055580777 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 -- 172.21.15.90:0/1055580777 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 msgr2=0x56001d91e1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d91e1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 -- 172.21.15.90:0/1055580777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d900780 msgr2=0x56001d900b60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d900780 0x56001d900b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 -- 172.21.15.90:0/1055580777 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56001d6d33b0 con 0x56001d9010a0 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f4257fff700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 0x7f42400a3850 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f4250009480 tx=0x7f425000b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.793+0000 7f42577fe700 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f424c009230 tx=0x7f424c09a780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c89a0e66d1e9fb62 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.794+0000 7f423e7fc700 1 -- 172.21.15.90:0/1055580777 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f424c0b1070 con 0x56001d9010a0 2024-04-06T14:07:08.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.794+0000 7f423e7fc700 1 -- 172.21.15.90:0/1055580777 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f424c0a2430 con 0x56001d9010a0 2024-04-06T14:07:08.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.794+0000 7f423e7fc700 1 -- 172.21.15.90:0/1055580777 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f424c09a210 con 0x56001d9010a0 2024-04-06T14:07:08.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.795+0000 7f426aafab80 1 -- 172.21.15.90:0/1055580777 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 msgr2=0x56001d91b8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.795+0000 7f426aafab80 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f424c009230 tx=0x7f424c09a780 comp rx=0 tx=0).stop 2024-04-06T14:07:08.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.796+0000 7f426aafab80 1 -- 172.21.15.90:0/1055580777 shutdown_connections 2024-04-06T14:07:08.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.796+0000 7f426aafab80 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d91e1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.796+0000 7f426aafab80 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d900780 0x56001d900b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.796+0000 7f426aafab80 1 --2- 172.21.15.90:0/1055580777 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d9010a0 0x56001d91b8b0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.796+0000 7f426aafab80 1 -- 172.21.15.90:0/1055580777 >> 172.21.15.90:0/1055580777 conn(0x56001d85f6a0 msgr2=0x56001d8fba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.797+0000 7f426aafab80 1 -- 172.21.15.90:0/1055580777 shutdown_connections 2024-04-06T14:07:08.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.797+0000 7f426aafab80 1 -- 172.21.15.90:0/1055580777 wait complete. 2024-04-06T14:07:08.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.797+0000 7f426aafab80 1 -- start start 2024-04-06T14:07:08.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.797+0000 7f426aafab80 1 Processor -- start 2024-04-06T14:07:08.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d919120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d91bdf0 0x56001d919660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d7edd80 con 0x56001d9010a0 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d7eda20 con 0x56001d91bdf0 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d6d56a0 con 0x56001d900780 2024-04-06T14:07:08.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.798+0000 7f426aafab80 10 client.0 _ll_register_callbacks cb 0x56001d7f7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.799+0000 7f4257fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.799+0000 7f425cee3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d91bdf0 0x56001d919660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.799+0000 7f4257fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:50812/0 (socket says 172.21.15.90:50812) 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.799+0000 7f4257fff700 1 -- 172.21.15.90:0/3429011441 learned_addr learned my addr 172.21.15.90:0/3429011441 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.799+0000 7f42577fe700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d919120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.800+0000 7f4257fff700 1 -- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d91bdf0 msgr2=0x56001d919660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.800+0000 7f4257fff700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d91bdf0 0x56001d919660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.800+0000 7f4257fff700 1 -- 172.21.15.90:0/3429011441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 msgr2=0x56001d919120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.800+0000 7f4257fff700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d919120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.800+0000 7f4257fff700 1 -- 172.21.15.90:0/3429011441 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42581b2cd0 con 0x56001d900780 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f4257fff700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f4250017440 tx=0x7f4250004ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6d9b06ac0378f46 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f42377fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:08.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4250021620 con 0x56001d900780 2024-04-06T14:07:08.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f425001c050 con 0x56001d900780 2024-04-06T14:07:08.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56001d812380 con 0x56001d900780 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4250021780 con 0x56001d900780 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f426aafab80 10 client.25297 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56001d67fb60 con 0x56001d900780 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.801+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56001d812380 con 0x56001d900780 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4250015900 con 0x56001d900780 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 10 client.25297 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 10 client.25297 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 20 client.25297 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d92bb80 0x56001d92dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 4 client.25297 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:08.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56001d84a700 con 0x56001d92bb80 2024-04-06T14:07:08.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f42577fe700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d92bb80 0x56001d92dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f42377fe700 10 client.25297 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:08.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.802+0000 7f42577fe700 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d92bb80 0x56001d92dfa0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3eb40b1f41bd1939 server_cookie=674cdef4b2514261 in_seq=0 out_seq=0 2024-04-06T14:07:08.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.803+0000 7f42377fe700 1 -- 172.21.15.90:0/3429011441 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56001d84a700 con 0x56001d92bb80 2024-04-06T14:07:08.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.803+0000 7f42377fe700 10 client.25297 handle_command_reply: tid=0 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.803+0000 7f426aafab80 1 client.25297 shutdown 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 msgr2=0x56001d918be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f4250017440 tx=0x7f4250004ad0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 shutdown_connections 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d91bdf0 0x56001d919660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d92bb80 0x56001d92dfa0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d900780 0x56001d918be0 secure :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f4250017440 tx=0x7f4250004ad0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 --2- 172.21.15.90:0/3429011441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d919120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f42377fe700 0 client.25297 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:08.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.804+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 >> 172.21.15.90:0/3429011441 conn(0x56001d85f6a0 msgr2=0x56001d8fac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.805+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 shutdown_connections 2024-04-06T14:07:08.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.805+0000 7f426aafab80 1 -- 172.21.15.90:0/3429011441 wait complete. 2024-04-06T14:07:08.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.805+0000 7f426aafab80 20 client.25297 trim_cache size 0 max 16384 2024-04-06T14:07:08.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.829+0000 7f426aafab80 1 Processor -- start 2024-04-06T14:07:08.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.829+0000 7f426aafab80 1 -- start start 2024-04-06T14:07:08.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d8fb520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d8fba60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d8fd220 con 0x56001d9010a0 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d91b750 con 0x56001d8f7890 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f426aafab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d901a10 con 0x56001d91bdf0 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47034/0 (socket says 172.21.15.90:47034) 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 -- 172.21.15.90:0/1283282585 learned_addr learned my addr 172.21.15.90:0/1283282585 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f42577fe700 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d8fb520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 -- 172.21.15.90:0/1283282585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 msgr2=0x56001d8fb520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d8fb520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 -- 172.21.15.90:0/1283282585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 msgr2=0x56001d8fba60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d8fba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.830+0000 7f4257fff700 1 -- 172.21.15.90:0/1283282585 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56001d817ea0 con 0x56001d9010a0 2024-04-06T14:07:08.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.831+0000 7f4257fff700 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f4250021040 tx=0x7f4250017ae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a1017b98ccaf75ec server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.832+0000 7f423effd700 1 -- 172.21.15.90:0/1283282585 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4250009560 con 0x56001d9010a0 2024-04-06T14:07:08.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.832+0000 7f423effd700 1 -- 172.21.15.90:0/1283282585 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f425000f040 con 0x56001d9010a0 2024-04-06T14:07:08.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.832+0000 7f423effd700 1 -- 172.21.15.90:0/1283282585 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4250014a90 con 0x56001d9010a0 2024-04-06T14:07:08.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.833+0000 7f426aafab80 1 -- 172.21.15.90:0/1283282585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 msgr2=0x56001d9194b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.833+0000 7f426aafab80 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 secure :-1 s=READY pgs=482 cs=0 l=1 rev1=1 crypto rx=0x7f4250021040 tx=0x7f4250017ae0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 -- 172.21.15.90:0/1283282585 shutdown_connections 2024-04-06T14:07:08.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d8fba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d9194b0 unknown :-1 s=CLOSED pgs=482 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 --2- 172.21.15.90:0/1283282585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d8fb520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 -- 172.21.15.90:0/1283282585 >> 172.21.15.90:0/1283282585 conn(0x56001d85f6a0 msgr2=0x56001d9174b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 -- 172.21.15.90:0/1283282585 shutdown_connections 2024-04-06T14:07:08.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 -- 172.21.15.90:0/1283282585 wait complete. 2024-04-06T14:07:08.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 -- start start 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.834+0000 7f426aafab80 1 Processor -- start 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d922ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d920fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d9214e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d921bc0 con 0x56001d9010a0 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d921d30 con 0x56001d8f7890 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56001d97f980 con 0x56001d91bdf0 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f426aafab80 10 client.0 _ll_register_callbacks cb 0x56001d7f7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f4257fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d922ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f4257fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d922ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:41950/0 (socket says 172.21.15.90:41950) 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f4257fff700 1 -- 172.21.15.90:0/1133282105 learned_addr learned my addr 172.21.15.90:0/1133282105 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:08.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f42577fe700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d920fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.835+0000 7f425cee3700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d9214e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 -- 172.21.15.90:0/1133282105 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 msgr2=0x56001d9214e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d9214e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 -- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 msgr2=0x56001d922ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d922ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 -- 172.21.15.90:0/1133282105 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56001d67fb60 con 0x56001d9010a0 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42577fe700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d920fa0 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f424c099f40 tx=0x7f424c09df60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c9b796cd4154619 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:08.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.836+0000 7f42377fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f424c0a7070 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f424c0b81e0 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f424c0a2430 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f424c0b1040 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 10 client.15717 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56001d6d7cd0 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f424c07d020 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f424c0aa030 con 0x56001d9010a0 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 10 client.15717 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 10 client.15717 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 20 client.15717 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:08.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d902a00 0x56001d904e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 4 client.15717 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.837+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56001d6a0cf0 con 0x56001d902a00 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f4257fff700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d902a00 0x56001d904e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f42377fe700 10 client.15717 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f4257fff700 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d902a00 0x56001d904e20 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b357c98cc3862ee9 server_cookie=695d5fd93d0dc2f in_seq=0 out_seq=0 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f42377fe700 1 -- 172.21.15.90:0/1133282105 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x56001d6a0cf0 con 0x56001d902a00 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f42377fe700 10 client.15717 handle_command_reply: tid=0 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 client.15717 shutdown 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 msgr2=0x56001d920fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d920fa0 secure :-1 s=READY pgs=483 cs=0 l=1 rev1=1 crypto rx=0x7f424c099f40 tx=0x7f424c09df60 comp rx=0 tx=0).stop 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 shutdown_connections 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56001d902a00 0x56001d904e20 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56001d91bdf0 0x56001d9214e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f42377fe700 0 client.15717 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56001d8f7890 0x56001d922ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 --2- 172.21.15.90:0/1133282105 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56001d9010a0 0x56001d920fa0 unknown :-1 s=CLOSED pgs=483 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 >> 172.21.15.90:0/1133282105 conn(0x56001d85f6a0 msgr2=0x56001d8fb500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.838+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 shutdown_connections 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f426aafab80 1 -- 172.21.15.90:0/1133282105 wait complete. 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f426aafab80 20 client.15717 trim_cache size 0 max 16384 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:07:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 msgr2=0x7f42400a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 0x7f42400a3850 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f4250009480 tx=0x7f425000b040 comp rx=0 tx=0).stop 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 msgr2=0x7f42581136f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f424c00e9a0 tx=0x7f424c00ecb0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 shutdown_connections 2024-04-06T14:07:08.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42400a1390 0x7f42400a3850 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4258112d00 0x7f4258113c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4258109750 0x7f42581131b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 --2- 172.21.15.90:0/1632390256 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f425810a070 0x7f42581136f0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 >> 172.21.15.90:0/1632390256 conn(0x7f425806be00 msgr2=0x7f4258070750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 shutdown_connections 2024-04-06T14:07:08.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:08.839+0000 7f425e946700 1 -- 172.21.15.90:0/1632390256 wait complete. 2024-04-06T14:07:08.856 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:08 smithi090 ceph-mon[43420]: pgmap v55: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 30 MiB/s wr, 1.86k op/s 2024-04-06T14:07:08.857 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:07:08.857 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:07:08.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:08 smithi134 ceph-mon[47549]: pgmap v55: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 30 MiB/s wr, 1.86k op/s 2024-04-06T14:07:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:08 smithi105 ceph-mon[46969]: pgmap v55: 97 pgs: 97 active+clean; 2.8 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 30 MiB/s wr, 1.86k op/s 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 -- 172.21.15.90:0/338266354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c1081b0 msgr2=0x7f7a0c10fd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 --2- 172.21.15.90:0/338266354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c10fd30 secure :-1 s=READY pgs=484 cs=0 l=1 rev1=1 crypto rx=0x7f7a0400b0a0 tx=0x7f7a0400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 -- 172.21.15.90:0/338266354 shutdown_connections 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 --2- 172.21.15.90:0/338266354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c10fd30 unknown :-1 s=CLOSED pgs=484 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 --2- 172.21.15.90:0/338266354 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c103090 0x7f7a0c103470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 --2- 172.21.15.90:0/338266354 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103a40 0x7f7a0c107a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 -- 172.21.15.90:0/338266354 >> 172.21.15.90:0/338266354 conn(0x7f7a0c0fe8d0 msgr2=0x7f7a0c100cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 -- 172.21.15.90:0/338266354 shutdown_connections 2024-04-06T14:07:09.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.051+0000 7f7a109b2700 1 -- 172.21.15.90:0/338266354 wait complete. 2024-04-06T14:07:09.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 Processor -- start 2024-04-06T14:07:09.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 -- start start 2024-04-06T14:07:09.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 0x7f7a0c199450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c199990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 0x7f7a0c19e1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a0c19e7b0 con 0x7f7a0c19dd30 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a0c19e920 con 0x7f7a0c1081b0 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a109b2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a0c19eae0 con 0x7f7a0c103090 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a0ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 0x7f7a0c19e1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a09d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c199990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a0a59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 0x7f7a0c199450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a0ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 0x7f7a0c19e1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47052/0 (socket says 172.21.15.90:47052) 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.052+0000 7f7a0ad9d700 1 -- 172.21.15.90:0/3260724626 learned_addr learned my addr 172.21.15.90:0/3260724626 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 -- 172.21.15.90:0/3260724626 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c1081b0 msgr2=0x7f7a0c199990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c199990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 -- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 msgr2=0x7f7a0c19e1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 0x7f7a0c19e1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 -- 172.21.15.90:0/3260724626 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79fc009520 con 0x7f7a0c103090 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a0a59c700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 0x7f7a0c199450 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f79fc00e9f0 tx=0x7f79fc00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79fc00cb70 con 0x7f7a0c103090 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a04009d00 con 0x7f7a0c103090 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a0c1a6380 con 0x7f7a0c103090 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f79fc00ee90 con 0x7f7a0c103090 2024-04-06T14:07:09.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.053+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f79fc003cf0 con 0x7f7a0c103090 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.055+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7a0c10b420 con 0x7f7a0c103090 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.058+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f79fc00ccd0 con 0x7f7a0c103090 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.058+0000 7f79fb7fe700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 0x7f79f40a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.058+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f79fc014070 con 0x7f7a0c103090 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.059+0000 7f7a09d9b700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 0x7f79f40a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.059+0000 7f7a09d9b700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 0x7f79f40a3810 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c19a9a0 tx=0x7f7a00009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.064+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f79fc0966e0 con 0x7f7a0c103090 2024-04-06T14:07:09.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.471+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f7a0c104560 con 0x7f7a0c103090 2024-04-06T14:07:09.472 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:07:09.472 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:07:09.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.472+0000 7f79fb7fe700 1 -- 172.21.15.90:0/3260724626 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f79fc099d00 con 0x7f7a0c103090 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.475+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 msgr2=0x7f79f40a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.475+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 0x7f79f40a3810 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c19a9a0 tx=0x7f7a00009260 comp rx=0 tx=0).stop 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.475+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 msgr2=0x7f7a0c199450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.475+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 0x7f7a0c199450 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f79fc00e9f0 tx=0x7f79fc00ed00 comp rx=0 tx=0).stop 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 shutdown_connections 2024-04-06T14:07:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f79f40a1350 0x7f79f40a3810 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a0c19dd30 0x7f7a0c19e1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a0c103090 0x7f7a0c199450 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 --2- 172.21.15.90:0/3260724626 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a0c1081b0 0x7f7a0c199990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 >> 172.21.15.90:0/3260724626 conn(0x7f7a0c0fe8d0 msgr2=0x7f7a0c100c00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 shutdown_connections 2024-04-06T14:07:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.476+0000 7f7a109b2700 1 -- 172.21.15.90:0/3260724626 wait complete. 2024-04-06T14:07:09.477 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:07:09.497 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:09.498 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:07:09.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.704+0000 7f28e0cc1700 1 -- 172.21.15.90:0/2556676818 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 msgr2=0x7f28dc069060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.704+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/2556676818 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc069060 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f28c4009990 tx=0x7f28c4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 -- 172.21.15.90:0/2556676818 shutdown_connections 2024-04-06T14:07:09.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/2556676818 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28dc103740 0x7f28dc103bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/2556676818 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc069060 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/2556676818 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28dc0695a0 0x7f28dc069a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 -- 172.21.15.90:0/2556676818 >> 172.21.15.90:0/2556676818 conn(0x7f28dc076d40 msgr2=0x7f28dc077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 -- 172.21.15.90:0/2556676818 shutdown_connections 2024-04-06T14:07:09.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.706+0000 7f28e0cc1700 1 -- 172.21.15.90:0/2556676818 wait complete. 2024-04-06T14:07:09.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 Processor -- start 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 -- start start 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28dc0695a0 0x7f28dc1a3620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28dc103740 0x7f28dc1a3b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28dc19ddb0 con 0x7f28dc0695a0 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28dc19df20 con 0x7f28dc103740 2024-04-06T14:07:09.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.707+0000 7f28e0cc1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28dc19e0e0 con 0x7f28dc105c40 2024-04-06T14:07:09.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:50872/0 (socket says 172.21.15.90:50872) 2024-04-06T14:07:09.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 -- 172.21.15.90:0/1173914121 learned_addr learned my addr 172.21.15.90:0/1173914121 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 -- 172.21.15.90:0/1173914121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28dc103740 msgr2=0x7f28dc1a3b60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28dc103740 0x7f28dc1a3b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 -- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28dc0695a0 msgr2=0x7f28dc1a3620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28dc0695a0 0x7f28dc1a3620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.708+0000 7f28dad9d700 1 -- 172.21.15.90:0/1173914121 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28c40095f0 con 0x7f28dc105c40 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28dad9d700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f28d400b970 tx=0x7f28d400bd30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28d37fe700 1 -- 172.21.15.90:0/1173914121 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28d400c9b0 con 0x7f28dc105c40 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28d37fe700 1 -- 172.21.15.90:0/1173914121 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28d4014440 con 0x7f28dc105c40 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28d37fe700 1 -- 172.21.15.90:0/1173914121 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28d40075d0 con 0x7f28dc105c40 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28dc19e3c0 con 0x7f28dc105c40 2024-04-06T14:07:09.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.709+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28dc06a9b0 con 0x7f28dc105c40 2024-04-06T14:07:09.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.713+0000 7f28ece75b80 1 Processor -- start 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.713+0000 7f28ece75b80 1 -- start start 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025c3b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025d01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d0251b990 con 0x560d025c3780 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d02519350 con 0x560d025cdde0 2024-04-06T14:07:09.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28ece75b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d02514c40 con 0x560d025c40a0 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28d9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025c3b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28da59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28d9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025c3b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47092/0 (socket says 172.21.15.90:47092) 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28da59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:50888/0 (socket says 172.21.15.90:50888) 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28d9d9b700 1 -- 172.21.15.90:0/248141932 learned_addr learned my addr 172.21.15.90:0/248141932 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.714+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025d01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 -- 172.21.15.90:0/248141932 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 msgr2=0x560d025d01d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025d01d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 -- 172.21.15.90:0/248141932 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 msgr2=0x560d025c3b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025c3b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 -- 172.21.15.90:0/248141932 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d023963b0 con 0x560d025c40a0 2024-04-06T14:07:09.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28da59c700 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f28c4009990 tx=0x7f28c4009f20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=80234d871ae59f3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28d37fe700 1 -- 172.21.15.90:0/1173914121 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f28d40145b0 con 0x7f28dc105c40 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.715+0000 7f28d0ff9700 1 -- 172.21.15.90:0/248141932 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28c401d070 con 0x560d025c40a0 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.716+0000 7f28d37fe700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 0x7f28c80a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.716+0000 7f28d37fe700 1 -- 172.21.15.90:0/1173914121 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28d4006e40 con 0x7f28dc105c40 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.716+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 0x7f28c80a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.717+0000 7f28d0ff9700 1 -- 172.21.15.90:0/248141932 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28c40050c0 con 0x560d025c40a0 2024-04-06T14:07:09.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.717+0000 7f28d0ff9700 1 -- 172.21.15.90:0/248141932 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28c4003a20 con 0x560d025c40a0 2024-04-06T14:07:09.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 -- 172.21.15.90:0/248141932 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 msgr2=0x560d025cd8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f28c4009990 tx=0x7f28c4009f20 comp rx=0 tx=0).stop 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 0x7f28c80a3920 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x560d025c51f0 tx=0x7f28cc00a250 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 -- 172.21.15.90:0/248141932 shutdown_connections 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025d01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025c3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 --2- 172.21.15.90:0/248141932 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025cd8a0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 -- 172.21.15.90:0/248141932 >> 172.21.15.90:0/248141932 conn(0x560d025226a0 msgr2=0x560d025bea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 -- 172.21.15.90:0/248141932 shutdown_connections 2024-04-06T14:07:09.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.718+0000 7f28ece75b80 1 -- 172.21.15.90:0/248141932 wait complete. 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 -- start start 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 Processor -- start 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025cab00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025cb0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d024b0d80 con 0x560d025cdde0 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d024b0a20 con 0x560d025c40a0 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d023986a0 con 0x560d025c3780 2024-04-06T14:07:09.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.719+0000 7f28ece75b80 10 client.0 _ll_register_callbacks cb 0x560d024ba470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:50900/0 (socket says 172.21.15.90:50900) 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 -- 172.21.15.90:0/3871597807 learned_addr learned my addr 172.21.15.90:0/3871597807 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025cab00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28dad9d700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025cb0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 -- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 msgr2=0x560d025cab00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025cab00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 -- 172.21.15.90:0/3871597807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 msgr2=0x560d025cb0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025cb0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 -- 172.21.15.90:0/3871597807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28dc114b40 con 0x560d025c3780 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28da59c700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f28c4006010 tx=0x7f28c4005510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dd90fcdb3d50ddb6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28c1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28c401d040 con 0x560d025c3780 2024-04-06T14:07:09.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28c4003bd0 con 0x560d025c3780 2024-04-06T14:07:09.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560d024d5380 con 0x560d025c3780 2024-04-06T14:07:09.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f28c4003d90 con 0x560d025c3780 2024-04-06T14:07:09.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28ece75b80 10 client.25321 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:09.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.720+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560d02342b60 con 0x560d025c3780 2024-04-06T14:07:09.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560d024d5380 con 0x560d025c3780 2024-04-06T14:07:09.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f28c4032070 con 0x560d025c3780 2024-04-06T14:07:09.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 10 client.25321 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 10 client.25321 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 20 client.25321 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d025dd480 0x560d025df8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 4 client.25321 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560d0250d700 con 0x560d025dd480 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d025dd480 0x560d025df8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d025dd480 0x560d025df8a0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a09397e8716852f6 server_cookie=e7dc8367dcffb262 in_seq=0 out_seq=0 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.721+0000 7f28c1ffb700 10 client.25321 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28c1ffb700 1 -- 172.21.15.90:0/3871597807 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560d0250d700 con 0x560d025dd480 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28c1ffb700 10 client.25321 handle_command_reply: tid=0 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 client.25321 shutdown 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 msgr2=0x560d025cd770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f28c4006010 tx=0x7f28c4005510 comp rx=0 tx=0).stop 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 shutdown_connections 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d025dd480 0x560d025df8a0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025cb0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025cd770 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 --2- 172.21.15.90:0/3871597807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025cab00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 >> 172.21.15.90:0/3871597807 conn(0x560d025226a0 msgr2=0x560d025bdc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28c1ffb700 0 client.25321 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 shutdown_connections 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 1 -- 172.21.15.90:0/3871597807 wait complete. 2024-04-06T14:07:09.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.722+0000 7f28ece75b80 20 client.25321 trim_cache size 0 max 16384 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 Processor -- start 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 -- start start 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025bfdc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025d4a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025ba890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d025cc090 con 0x560d025c3780 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d025c44b0 con 0x560d025cdde0 2024-04-06T14:07:09.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.742+0000 7f28ece75b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d025c4a60 con 0x560d025c40a0 2024-04-06T14:07:09.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28d9d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025d4a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28dad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025ba890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28dad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025ba890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42000/0 (socket says 172.21.15.90:42000) 2024-04-06T14:07:09.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28dad9d700 1 -- 172.21.15.90:0/645437396 learned_addr learned my addr 172.21.15.90:0/645437396 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28da59c700 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025bfdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28d9d9b700 1 -- 172.21.15.90:0/645437396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 msgr2=0x560d025ba890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025ba890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28d9d9b700 1 -- 172.21.15.90:0/645437396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 msgr2=0x560d025bfdc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.743+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025bfdc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.744+0000 7f28d9d9b700 1 -- 172.21.15.90:0/645437396 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560d024daea0 con 0x560d025c40a0 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.744+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025d4a60 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x560d025cc030 tx=0x7f28cc013000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a55e8af82e73ca81 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.744+0000 7f28d17fa700 1 -- 172.21.15.90:0/645437396 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28cc012d40 con 0x560d025c40a0 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.744+0000 7f28d17fa700 1 -- 172.21.15.90:0/645437396 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28cc026040 con 0x560d025c40a0 2024-04-06T14:07:09.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.744+0000 7f28d17fa700 1 -- 172.21.15.90:0/645437396 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28cc026a00 con 0x560d025c40a0 2024-04-06T14:07:09.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.745+0000 7f28ece75b80 1 -- 172.21.15.90:0/645437396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 msgr2=0x560d025d4a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.745+0000 7f28ece75b80 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025d4a60 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x560d025cc030 tx=0x7f28cc013000 comp rx=0 tx=0).stop 2024-04-06T14:07:09.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 -- 172.21.15.90:0/645437396 shutdown_connections 2024-04-06T14:07:09.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025cdde0 0x560d025ba890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025c3780 0x560d025bfdc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 --2- 172.21.15.90:0/645437396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c40a0 0x560d025d4a60 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 -- 172.21.15.90:0/645437396 >> 172.21.15.90:0/645437396 conn(0x560d025226a0 msgr2=0x560d025ca390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.746+0000 7f28ece75b80 1 -- 172.21.15.90:0/645437396 shutdown_connections 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 -- 172.21.15.90:0/645437396 wait complete. 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 -- start start 2024-04-06T14:07:09.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 Processor -- start 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025d4880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025d32c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d025d39d0 con 0x560d025cdde0 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d025d3b40 con 0x560d025c40a0 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.747+0000 7f28ece75b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560d026312c0 con 0x560d025c3780 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28ece75b80 10 client.0 _ll_register_callbacks cb 0x560d024ba470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42002/0 (socket says 172.21.15.90:42002) 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 -- 172.21.15.90:0/2110608649 learned_addr learned my addr 172.21.15.90:0/2110608649 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28da59c700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025d4880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28dad9d700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025d32c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 -- 172.21.15.90:0/2110608649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 msgr2=0x560d025d4880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025d4880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 -- 172.21.15.90:0/2110608649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 msgr2=0x560d025d32c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025d32c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.748+0000 7f28d9d9b700 1 -- 172.21.15.90:0/2110608649 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28cc010c90 con 0x560d025c40a0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28d9d9b700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x560d025cc030 tx=0x7f28cc0045f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ea54b65615ce795c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28c1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28cc020430 con 0x560d025c40a0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f28c40050c0 con 0x560d025c40a0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28cc020590 con 0x560d025c40a0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f28cc020750 con 0x560d025c40a0 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28ece75b80 10 client.25397 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:09.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.749+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560d0239acd0 con 0x560d025c40a0 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28cc043020 con 0x560d025c40a0 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f28cc02c030 con 0x560d025c40a0 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 10 client.25397 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 10 client.25397 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 20 client.25397 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:09.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d026328d0 0x560d02634cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 4 client.25397 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x560d02363cf0 con 0x560d026328d0 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28da59c700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d026328d0 0x560d02634cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28c1ffb700 10 client.25397 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.750+0000 7f28da59c700 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d026328d0 0x560d02634cf0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a4fb2bd23809d71f server_cookie=3455bc5400282543 in_seq=0 out_seq=0 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28c1ffb700 1 -- 172.21.15.90:0/2110608649 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x560d02363cf0 con 0x560d026328d0 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28c1ffb700 10 client.25397 handle_command_reply: tid=0 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 client.25397 shutdown 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 msgr2=0x560d025d2d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x560d025cc030 tx=0x7f28cc0045f0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 shutdown_connections 2024-04-06T14:07:09.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x560d026328d0 0x560d02634cf0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560d025cdde0 0x560d025d32c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560d025c3780 0x560d025d4880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 --2- 172.21.15.90:0/2110608649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560d025c40a0 0x560d025d2d80 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 >> 172.21.15.90:0/2110608649 conn(0x560d025226a0 msgr2=0x560d025ca300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28c1ffb700 0 client.25397 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 shutdown_connections 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 1 -- 172.21.15.90:0/2110608649 wait complete. 2024-04-06T14:07:09.754 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.751+0000 7f28ece75b80 20 client.25397 trim_cache size 0 max 16384 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.752+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 msgr2=0x7f28c80a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.752+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 0x7f28c80a3920 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x560d025c51f0 tx=0x7f28cc00a250 comp rx=0 tx=0).stop 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.752+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 msgr2=0x7f28dc19d750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.752+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f28d400b970 tx=0x7f28d400bd30 comp rx=0 tx=0).stop 2024-04-06T14:07:09.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 shutdown_connections 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28c80a1460 0x7f28c80a3920 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28dc105c40 0x7f28dc19d750 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28dc0695a0 0x7f28dc1a3620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 --2- 172.21.15.90:0/1173914121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28dc103740 0x7f28dc1a3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 >> 172.21.15.90:0/1173914121 conn(0x7f28dc076d40 msgr2=0x7f28dc102720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 shutdown_connections 2024-04-06T14:07:09.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.753+0000 7f28e0cc1700 1 -- 172.21.15.90:0/1173914121 wait complete. 2024-04-06T14:07:09.768 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3260724626' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:09.770 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:09.770 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:07:09.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3260724626' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:09.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 -- 172.21.15.90:0/467543534 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb4068aa0 msgr2=0x7f1bb4105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 --2- 172.21.15.90:0/467543534 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb4068aa0 0x7f1bb4105970 secure :-1 s=READY pgs=485 cs=0 l=1 rev1=1 crypto rx=0x7f1ba40099c0 tx=0x7f1ba4009cd0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 -- 172.21.15.90:0/467543534 shutdown_connections 2024-04-06T14:07:09.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 --2- 172.21.15.90:0/467543534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb410dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 --2- 172.21.15.90:0/467543534 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb40680f0 0x7f1bb40684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 --2- 172.21.15.90:0/467543534 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb4068aa0 0x7f1bb4105970 unknown :-1 s=CLOSED pgs=485 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 -- 172.21.15.90:0/467543534 >> 172.21.15.90:0/467543534 conn(0x7f1bb4076d50 msgr2=0x7f1bb4077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 -- 172.21.15.90:0/467543534 shutdown_connections 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.939+0000 7f1bbb315700 1 -- 172.21.15.90:0/467543534 wait complete. 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 Processor -- start 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 -- start start 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb41976b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb419ba50 0x7f1bb419bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bb419c4d0 con 0x7f1bb40680f0 2024-04-06T14:07:09.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bb419c640 con 0x7f1bb41060a0 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bbb315700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bb419c800 con 0x7f1bb419ba50 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47128/0 (socket says 172.21.15.90:47128) 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 -- 172.21.15.90:0/3139556506 learned_addr learned my addr 172.21.15.90:0/3139556506 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb88b0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb41976b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb419ba50 0x7f1bb419bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 -- 172.21.15.90:0/3139556506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb419ba50 msgr2=0x7f1bb419bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb419ba50 0x7f1bb419bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.940+0000 7f1bb90b1700 1 -- 172.21.15.90:0/3139556506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 msgr2=0x7f1bb41976b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb41976b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bb90b1700 1 -- 172.21.15.90:0/3139556506 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1ba40095f0 con 0x7f1bb40680f0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb41976b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f1bb000d950 tx=0x7f1bb000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1baa7fc700 1 -- 172.21.15.90:0/3139556506 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb0009940 con 0x7f1bb40680f0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1baa7fc700 1 -- 172.21.15.90:0/3139556506 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb0010460 con 0x7f1bb40680f0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1bb41a3d60 con 0x7f1bb40680f0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1baa7fc700 1 -- 172.21.15.90:0/3139556506 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb000f620 con 0x7f1bb40680f0 2024-04-06T14:07:09.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.941+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bb41a4230 con 0x7f1bb40680f0 2024-04-06T14:07:09.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.943+0000 7f1baa7fc700 1 -- 172.21.15.90:0/3139556506 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1bb00108d0 con 0x7f1bb40680f0 2024-04-06T14:07:09.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.943+0000 7f1baa7fc700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 0x7f1ba00a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.943+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 0x7f1ba00a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.943+0000 7f1baa7fc700 1 -- 172.21.15.90:0/3139556506 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bb00cce40 con 0x7f1bb40680f0 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.944+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 0x7f1ba00a37d0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f1ba4006010 tx=0x7f1ba4000bc0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.944+0000 7f1bc74c9b80 1 Processor -- start 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 -- start start 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d2cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d55e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b551c990 con 0x5616b55c4780 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b551a350 con 0x5616b55c50a0 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b5515c40 con 0x5616b55d31f0 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47132/0 (socket says 172.21.15.90:47132) 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 -- 172.21.15.90:0/580589701 learned_addr learned my addr 172.21.15.90:0/580589701 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d55e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d2cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 -- 172.21.15.90:0/580589701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 msgr2=0x5616b55d55e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d55e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 -- 172.21.15.90:0/580589701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 msgr2=0x5616b55d2cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d2cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 -- 172.21.15.90:0/580589701 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5616b53973b0 con 0x5616b55c4780 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d2cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f1bb00a1eb0 tx=0x7f1bb00999f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e272c9753248d53c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1b9f7fe700 1 -- 172.21.15.90:0/580589701 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb00a23e0 con 0x5616b55c4780 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1b9f7fe700 1 -- 172.21.15.90:0/580589701 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb00ca820 con 0x5616b55c4780 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.945+0000 7f1b9f7fe700 1 -- 172.21.15.90:0/580589701 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb009a400 con 0x5616b55c4780 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/580589701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 msgr2=0x5616b55c4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 secure :-1 s=READY pgs=487 cs=0 l=1 rev1=1 crypto rx=0x7f1bb00a1eb0 tx=0x7f1bb00999f0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/580589701 shutdown_connections 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d55e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=CLOSED pgs=487 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/580589701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d2cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/580589701 >> 172.21.15.90:0/580589701 conn(0x5616b55236a0 msgr2=0x5616b55bfa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/580589701 shutdown_connections 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.946+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/580589701 wait complete. 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 -- start start 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 Processor -- start 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55cfeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d0530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b54b1d80 con 0x5616b55c4780 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b54b1a20 con 0x5616b55c50a0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b53996a0 con 0x5616b55d31f0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bc74c9b80 10 client.0 _ll_register_callbacks cb 0x5616b54bb470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47146/0 (socket says 172.21.15.90:47146) 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 -- 172.21.15.90:0/567695512 learned_addr learned my addr 172.21.15.90:0/567695512 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55cfeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d0530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 -- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 msgr2=0x5616b55d0530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d0530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 -- 172.21.15.90:0/567695512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 msgr2=0x5616b55cfeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55cfeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.947+0000 7f1bb90b1700 1 -- 172.21.15.90:0/567695512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bb41a4230 con 0x5616b55c4780 2024-04-06T14:07:09.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f1bb00969d0 tx=0x7f1bb00963d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9b3f530447f7d32 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb00ca990 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bb00a23e0 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5616b54d6380 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1bb0099660 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 10 client.15747 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5616b5343b60 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5616b54d6380 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1bb00a1f50 con 0x5616b55c4780 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 10 client.15747 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 10 client.15747 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 20 client.15747 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:09.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b55e2880 0x5616b55e4c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 4 client.15747 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.948+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5616b550e700 con 0x5616b55e2880 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.949+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b55e2880 0x5616b55e4c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.949+0000 7f1b9cff9700 10 client.15747 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.949+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b55e2880 0x5616b55e4c70 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ed97018973739417 server_cookie=dffd63376909566 in_seq=0 out_seq=0 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1b9cff9700 1 -- 172.21.15.90:0/567695512 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5616b550e700 con 0x5616b55e2880 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1b9cff9700 10 client.15747 handle_command_reply: tid=0 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 client.15747 shutdown 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 msgr2=0x5616b55d2bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 secure :-1 s=READY pgs=488 cs=0 l=1 rev1=1 crypto rx=0x7f1bb00969d0 tx=0x7f1bb00963d0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 shutdown_connections 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b55e2880 0x5616b55e4c70 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d0530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d2bc0 unknown :-1 s=CLOSED pgs=488 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/567695512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55cfeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1b9cff9700 0 client.15747 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 >> 172.21.15.90:0/567695512 conn(0x5616b55236a0 msgr2=0x5616b55bec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 shutdown_connections 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/567695512 wait complete. 2024-04-06T14:07:09.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.950+0000 7f1bc74c9b80 20 client.15747 trim_cache size 0 max 16384 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 Processor -- start 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 -- start start 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d9e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b55b9d50 con 0x5616b55d31f0 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b55c54b0 con 0x5616b55c50a0 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b55df940 con 0x5616b55c4780 2024-04-06T14:07:09.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47154/0 (socket says 172.21.15.90:47154) 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 -- 172.21.15.90:0/1449465223 learned_addr learned my addr 172.21.15.90:0/1449465223 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d9e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 -- 172.21.15.90:0/1449465223 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55c4780 msgr2=0x5616b55c4b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 -- 172.21.15.90:0/1449465223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 msgr2=0x5616b55d9e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d9e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.963+0000 7f1bb98b2700 1 -- 172.21.15.90:0/1449465223 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5616b54dbea0 con 0x5616b55d31f0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f1bac014440 tx=0x7f1bac014750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=58020ce9fd34da83 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1b9ffff700 1 -- 172.21.15.90:0/1449465223 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bac01f460 con 0x5616b55d31f0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1b9ffff700 1 -- 172.21.15.90:0/1449465223 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bac01fa80 con 0x5616b55d31f0 2024-04-06T14:07:09.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1b9ffff700 1 -- 172.21.15.90:0/1449465223 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bac016b00 con 0x5616b55d31f0 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/1449465223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 msgr2=0x5616b55bf520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 secure :-1 s=READY pgs=489 cs=0 l=1 rev1=1 crypto rx=0x7f1bac014440 tx=0x7f1bac014750 comp rx=0 tx=0).stop 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/1449465223 shutdown_connections 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55d31f0 0x5616b55bf520 unknown :-1 s=CLOSED pgs=489 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55c4780 0x5616b55c4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/1449465223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d9e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/1449465223 >> 172.21.15.90:0/1449465223 conn(0x5616b55236a0 msgr2=0x5616b55ce780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/1449465223 shutdown_connections 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.964+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/1449465223 wait complete. 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 -- start start 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 Processor -- start 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d8200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d8760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b55d8e70 con 0x5616b55c4780 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b55d8fe0 con 0x5616b55c50a0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5616b5636690 con 0x5616b55d31f0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bc74c9b80 10 client.0 _ll_register_callbacks cb 0x5616b54bb470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47168/0 (socket says 172.21.15.90:47168) 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 -- 172.21.15.90:0/696214297 learned_addr learned my addr 172.21.15.90:0/696214297 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d8200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb98b2700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d8760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 -- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 msgr2=0x5616b55d8760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d8760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 -- 172.21.15.90:0/696214297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 msgr2=0x5616b55d8200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:09.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d8200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.965+0000 7f1bb90b1700 1 -- 172.21.15.90:0/696214297 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bac00f040 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bb90b1700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f1bb0099ef0 tx=0x7f1bb009a0a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4d33d5c85762bb3f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb0099660 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bb009a660 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1bb00ca820 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1bb00a23e0 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 10 client.15753 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5616b539bcd0 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb00a6020 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1bb00054b0 con 0x5616b55c4780 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 10 client.15753 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 10 client.15753 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:07:09.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 20 client.15753 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b5637a00 0x5616b5639df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 4 client.15753 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5616b5364cf0 con 0x5616b5637a00 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.966+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b5637a00 0x5616b5639df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bb88b0700 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b5637a00 0x5616b5639df0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=692088ea097905b8 server_cookie=15734c34b3537dfa in_seq=0 out_seq=0 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1b9cff9700 10 client.15753 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1b9cff9700 1 -- 172.21.15.90:0/696214297 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5616b5364cf0 con 0x5616b5637a00 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1b9cff9700 10 client.15753 handle_command_reply: tid=0 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 client.15753 shutdown 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 msgr2=0x5616b55d9cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 secure :-1 s=READY pgs=490 cs=0 l=1 rev1=1 crypto rx=0x7f1bb0099ef0 tx=0x7f1bb009a0a0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 shutdown_connections 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5616b5637a00 0x5616b5639df0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5616b55d31f0 0x5616b55d8760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5616b55c4780 0x5616b55d9cc0 unknown :-1 s=CLOSED pgs=490 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1b9cff9700 0 client.15753 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 --2- 172.21.15.90:0/696214297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5616b55c50a0 0x5616b55d8200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 >> 172.21.15.90:0/696214297 conn(0x5616b55236a0 msgr2=0x5616b55ce610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 shutdown_connections 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 1 -- 172.21.15.90:0/696214297 wait complete. 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.967+0000 7f1bc74c9b80 20 client.15753 trim_cache size 0 max 16384 2024-04-06T14:07:09.974 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:09.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 msgr2=0x7f1ba00a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 0x7f1ba00a37d0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f1ba4006010 tx=0x7f1ba4000bc0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 msgr2=0x7f1bb4197170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 secure :-1 s=READY pgs=486 cs=0 l=1 rev1=1 crypto rx=0x7f1bb000d950 tx=0x7f1bb000dc60 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 shutdown_connections 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ba00a1310 0x7f1ba00a37d0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bb419ba50 0x7f1bb419bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bb40680f0 0x7f1bb4197170 unknown :-1 s=CLOSED pgs=486 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 --2- 172.21.15.90:0/3139556506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bb41060a0 0x7f1bb41976b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 >> 172.21.15.90:0/3139556506 conn(0x7f1bb4076d50 msgr2=0x7f1bb40fdf60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 shutdown_connections 2024-04-06T14:07:09.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:09.968+0000 7f1bbb315700 1 -- 172.21.15.90:0/3139556506 wait complete. 2024-04-06T14:07:09.979 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:09.979 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:07:09.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3260724626' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:10.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.155+0000 7f63bc152700 1 -- 172.21.15.90:0/3590637416 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b40ffe70 msgr2=0x7f63b4100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.155+0000 7f63bc152700 1 --2- 172.21.15.90:0/3590637416 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b40ffe70 0x7f63b4100250 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f63b0009990 tx=0x7f63b0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 -- 172.21.15.90:0/3590637416 shutdown_connections 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 --2- 172.21.15.90:0/3590637416 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b4104f90 0x7f63b4069260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 --2- 172.21.15.90:0/3590637416 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b40ffe70 0x7f63b4100250 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 --2- 172.21.15.90:0/3590637416 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b4100820 0x7f63b4104860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 -- 172.21.15.90:0/3590637416 >> 172.21.15.90:0/3590637416 conn(0x7f63b4076d80 msgr2=0x7f63b4077190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 -- 172.21.15.90:0/3590637416 shutdown_connections 2024-04-06T14:07:10.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.156+0000 7f63bc152700 1 -- 172.21.15.90:0/3590637416 wait complete. 2024-04-06T14:07:10.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 Processor -- start 2024-04-06T14:07:10.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 -- start start 2024-04-06T14:07:10.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b4104f90 0x7f63b410c630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b419f3c0 0x7f63b419f840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63b419fea0 con 0x7f63b419f3c0 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63b41a0010 con 0x7f63b40ffe70 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63bc152700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f63b41a01d0 con 0x7f63b4104f90 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42060/0 (socket says 172.21.15.90:42060) 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.157+0000 7f63b9eee700 1 -- 172.21.15.90:0/2342928092 learned_addr learned my addr 172.21.15.90:0/2342928092 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b419f3c0 0x7f63b419f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b96ed700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b4104f90 0x7f63b410c630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 -- 172.21.15.90:0/2342928092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b4104f90 msgr2=0x7f63b410c630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b4104f90 0x7f63b410c630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 -- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b419f3c0 msgr2=0x7f63b419f840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b419f3c0 0x7f63b419f840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 -- 172.21.15.90:0/2342928092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63b00095f0 con 0x7f63b40ffe70 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63b9eee700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f63b0005910 tx=0x7f63b0004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63aaffd700 1 -- 172.21.15.90:0/2342928092 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63b001d070 con 0x7f63b40ffe70 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63aaffd700 1 -- 172.21.15.90:0/2342928092 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f63b0022470 con 0x7f63b40ffe70 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.158+0000 7f63aaffd700 1 -- 172.21.15.90:0/2342928092 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63b0017650 con 0x7f63b40ffe70 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.159+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63b41120c0 con 0x7f63b40ffe70 2024-04-06T14:07:10.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.159+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63b4112520 con 0x7f63b40ffe70 2024-04-06T14:07:10.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.163+0000 7f63aaffd700 1 -- 172.21.15.90:0/2342928092 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f63b00225e0 con 0x7f63b40ffe70 2024-04-06T14:07:10.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.163+0000 7f63c8306b80 1 Processor -- start 2024-04-06T14:07:10.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 -- start start 2024-04-06T14:07:10.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c14b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c1ede0 0x55b346c211d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346b6c990 con 0x55b346c1ede0 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346b6a350 con 0x55b346c14780 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63c8306b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346b65c40 con 0x55b346c150a0 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63aaffd700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 0x7f63a00a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51026/0 (socket says 172.21.15.90:51026) 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63b9eee700 1 -- 172.21.15.90:0/567006443 learned_addr learned my addr 172.21.15.90:0/567006443 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.164+0000 7f63b96ed700 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c1ede0 0x55b346c211d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63aaffd700 1 -- 172.21.15.90:0/2342928092 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f63b00ce030 con 0x7f63b40ffe70 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 -- 172.21.15.90:0/567006443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 msgr2=0x55b346c14b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c14b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 -- 172.21.15.90:0/567006443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c1ede0 msgr2=0x55b346c211d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c1ede0 0x55b346c211d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 -- 172.21.15.90:0/567006443 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b3469e73b0 con 0x55b346c150a0 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 0x7f63a00a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.165+0000 7f63b9eee700 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f63b00976e0 tx=0x7f63b0009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1027ed4219f3ad4d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f639ffff700 1 -- 172.21.15.90:0/567006443 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63b009ab40 con 0x55b346c150a0 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f639ffff700 1 -- 172.21.15.90:0/567006443 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f63b009eb80 con 0x55b346c150a0 2024-04-06T14:07:10.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f639ffff700 1 -- 172.21.15.90:0/567006443 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63b00228c0 con 0x55b346c150a0 2024-04-06T14:07:10.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f63c8306b80 1 -- 172.21.15.90:0/567006443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 msgr2=0x55b346c1e8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 0x7f63a00a37d0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f63ac00a5b0 tx=0x7f63ac00c040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.166+0000 7f63c8306b80 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f63b00976e0 tx=0x7f63b0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 -- 172.21.15.90:0/567006443 shutdown_connections 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c1ede0 0x55b346c211d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c14b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 --2- 172.21.15.90:0/567006443 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1e8a0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 -- 172.21.15.90:0/567006443 >> 172.21.15.90:0/567006443 conn(0x55b346b736a0 msgr2=0x55b346c0fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 -- 172.21.15.90:0/567006443 shutdown_connections 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 -- 172.21.15.90:0/567006443 wait complete. 2024-04-06T14:07:10.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 -- start start 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.167+0000 7f63c8306b80 1 Processor -- start 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c1e860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c1bba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346b01d80 con 0x55b346c150a0 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346b01a20 con 0x55b346c14780 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b3469e96a0 con 0x55b346c1ede0 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63c8306b80 10 client.0 _ll_register_callbacks cb 0x55b346b0b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63ba6ef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63ba6ef700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51032/0 (socket says 172.21.15.90:51032) 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.168+0000 7f63ba6ef700 1 -- 172.21.15.90:0/631509278 learned_addr learned my addr 172.21.15.90:0/631509278 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:10.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 -- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 msgr2=0x55b346c1e860 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c1e860 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 -- 172.21.15.90:0/631509278 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 msgr2=0x55b346c1bba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c1bba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 -- 172.21.15.90:0/631509278 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63b4112520 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f63ac005340 tx=0x7f63ac004ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2609243dac68c5a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f639d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63ac016670 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b346b26380 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f63ac0167d0 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f63ac016990 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63c8306b80 10 client.25357 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.169+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b346993b60 con 0x55b346c1ede0 2024-04-06T14:07:10.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b346b26380 con 0x55b346c1ede0 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f63ac020c80 con 0x55b346c1ede0 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 10 client.25357 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 10 client.25357 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 20 client.25357 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c2e810 0x55b346c30c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 4 client.25357 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.170+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b346b5e700 con 0x55b346c2e810 2024-04-06T14:07:10.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.172+0000 7f63b9eee700 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c2e810 0x55b346c30c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.172+0000 7f63b9eee700 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c2e810 0x55b346c30c30 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1bff6c35cc3547fb server_cookie=fe54dfe1731cf2b3 in_seq=0 out_seq=0 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.172+0000 7f639d7fa700 10 client.25357 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f639d7fa700 1 -- 172.21.15.90:0/631509278 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b346b5e700 con 0x55b346c2e810 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f639d7fa700 10 client.25357 handle_command_reply: tid=0 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 client.25357 shutdown 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 msgr2=0x55b346c1c290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f63ac005340 tx=0x7f63ac004ad0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 shutdown_connections 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c2e810 0x55b346c30c30 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c1ede0 0x55b346c1c290 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c14780 0x55b346c1e860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 --2- 172.21.15.90:0/631509278 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c1bba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 >> 172.21.15.90:0/631509278 conn(0x55b346b736a0 msgr2=0x55b346c0ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f639d7fa700 0 client.25357 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:10.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 shutdown_connections 2024-04-06T14:07:10.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 1 -- 172.21.15.90:0/631509278 wait complete. 2024-04-06T14:07:10.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.173+0000 7f63c8306b80 20 client.25357 trim_cache size 0 max 16384 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 Processor -- start 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 -- start start 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 0x55b346c1bb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1c1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c257d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c2be40 con 0x55b346c14780 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c1d150 con 0x55b346c1ede0 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63c8306b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c15a10 con 0x55b346c150a0 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63b96ed700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1c1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 0x55b346c1bb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63b96ed700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1c1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51034/0 (socket says 172.21.15.90:51034) 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.191+0000 7f63b96ed700 1 -- 172.21.15.90:0/393216194 learned_addr learned my addr 172.21.15.90:0/393216194 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c257d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 -- 172.21.15.90:0/393216194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 msgr2=0x55b346c1c1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1c1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 -- 172.21.15.90:0/393216194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 msgr2=0x55b346c1bb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 0x55b346c1bb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 -- 172.21.15.90:0/393216194 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b346b2bea0 con 0x55b346c1ede0 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63b9eee700 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 0x55b346c1bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c257d0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f63ac005a90 tx=0x7f63ac00a410 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fbf0aa227aa4aa09 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63a8ff9700 1 -- 172.21.15.90:0/393216194 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63ac014ac0 con 0x55b346c1ede0 2024-04-06T14:07:10.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63a8ff9700 1 -- 172.21.15.90:0/393216194 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f63ac010040 con 0x55b346c1ede0 2024-04-06T14:07:10.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.192+0000 7f63a8ff9700 1 -- 172.21.15.90:0/393216194 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63ac020bd0 con 0x55b346c1ede0 2024-04-06T14:07:10.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.193+0000 7f63c8306b80 1 -- 172.21.15.90:0/393216194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 msgr2=0x55b346c257d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.193+0000 7f63c8306b80 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c257d0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f63ac005a90 tx=0x7f63ac00a410 comp rx=0 tx=0).stop 2024-04-06T14:07:10.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.195+0000 7f63c8306b80 1 -- 172.21.15.90:0/393216194 shutdown_connections 2024-04-06T14:07:10.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.195+0000 7f63c8306b80 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c257d0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.195+0000 7f63c8306b80 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c14780 0x55b346c1bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.195+0000 7f63c8306b80 1 --2- 172.21.15.90:0/393216194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c150a0 0x55b346c1c1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.195+0000 7f63c8306b80 1 -- 172.21.15.90:0/393216194 >> 172.21.15.90:0/393216194 conn(0x55b346b736a0 msgr2=0x55b346c1b430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.196+0000 7f63c8306b80 1 -- 172.21.15.90:0/393216194 shutdown_connections 2024-04-06T14:07:10.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.196+0000 7f63c8306b80 1 -- 172.21.15.90:0/393216194 wait complete. 2024-04-06T14:07:10.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.196+0000 7f63c8306b80 1 -- start start 2024-04-06T14:07:10.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.196+0000 7f63c8306b80 1 Processor -- start 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 0x55b346c25230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c23860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c23f40 con 0x55b346c150a0 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c240b0 con 0x55b346c1ede0 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b346c24270 con 0x55b346c14780 2024-04-06T14:07:10.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.197+0000 7f63c8306b80 10 client.0 _ll_register_callbacks cb 0x55b346b0b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47218/0 (socket says 172.21.15.90:47218) 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 -- 172.21.15.90:0/981686734 learned_addr learned my addr 172.21.15.90:0/981686734 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b9eee700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 0x55b346c25230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63ba6ef700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c23860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 -- 172.21.15.90:0/981686734 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 msgr2=0x55b346c25230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 0x55b346c25230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 -- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 msgr2=0x55b346c23860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c23860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 -- 172.21.15.90:0/981686734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55b3469e73b0 con 0x55b346c150a0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b9eee700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 0x55b346c25230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f63b96ed700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f63a40128d0 tx=0x7f63a4012be0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9d13dea692b03df1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f639d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f63a40119d0 con 0x55b346c150a0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.198+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f63a4011b30 con 0x55b346c150a0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f63b009eb80 con 0x55b346c150a0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f63a4011cf0 con 0x55b346c150a0 2024-04-06T14:07:10.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 10 client.15765 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b346993b60 con 0x55b346c150a0 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f63b009eb80 con 0x55b346c150a0 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f63a4024070 con 0x55b346c150a0 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 10 client.15765 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 10 client.15765 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 20 client.15765 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c839c0 0x55b346c85de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 4 client.15765 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:10.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b3469b4cf0 con 0x55b346c839c0 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.199+0000 7f63b9eee700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c839c0 0x55b346c85de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.200+0000 7f639d7fa700 10 client.15765 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.200+0000 7f63b9eee700 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c839c0 0x55b346c85de0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d277a1af8e54f39b server_cookie=8aae9b82700f1505 in_seq=0 out_seq=0 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.200+0000 7f639d7fa700 1 -- 172.21.15.90:0/981686734 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b3469b4cf0 con 0x55b346c839c0 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.200+0000 7f639d7fa700 10 client.15765 handle_command_reply: tid=0 2024-04-06T14:07:10.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.200+0000 7f63c8306b80 1 client.15765 shutdown 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 msgr2=0x55b346c25770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 secure :-1 s=READY pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f63a40128d0 tx=0x7f63a4012be0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 shutdown_connections 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b346c1ede0 0x55b346c23860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b346c839c0 0x55b346c85de0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b346c14780 0x55b346c25230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 --2- 172.21.15.90:0/981686734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b346c150a0 0x55b346c25770 secure :-1 s=CLOSED pgs=491 cs=0 l=1 rev1=1 crypto rx=0x7f63a40128d0 tx=0x7f63a4012be0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 >> 172.21.15.90:0/981686734 conn(0x55b346b736a0 msgr2=0x55b346c2a7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.201+0000 7f639d7fa700 0 client.15765 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:10.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.202+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 shutdown_connections 2024-04-06T14:07:10.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.202+0000 7f63c8306b80 1 -- 172.21.15.90:0/981686734 wait complete. 2024-04-06T14:07:10.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.202+0000 7f63c8306b80 20 client.15765 trim_cache size 0 max 16384 2024-04-06T14:07:10.206 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:10.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 msgr2=0x7f63a00a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 0x7f63a00a37d0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f63ac00a5b0 tx=0x7f63ac00c040 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 msgr2=0x7f63b410c0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f63b0005910 tx=0x7f63b0004730 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 shutdown_connections 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f63a00a1310 0x7f63a00a37d0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f63b419f3c0 0x7f63b419f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.203+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f63b40ffe70 0x7f63b410c0f0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.204+0000 7f63bc152700 1 --2- 172.21.15.90:0/2342928092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f63b4104f90 0x7f63b410c630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.204+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 >> 172.21.15.90:0/2342928092 conn(0x7f63b4076d80 msgr2=0x7f63b40feb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.204+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 shutdown_connections 2024-04-06T14:07:10.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:10.204+0000 7f63bc152700 1 -- 172.21.15.90:0/2342928092 wait complete. 2024-04-06T14:07:10.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:10 smithi134 ceph-mon[47549]: pgmap v56: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 37 MiB/s wr, 2.22k op/s 2024-04-06T14:07:10.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:10 smithi105 ceph-mon[46969]: pgmap v56: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 37 MiB/s wr, 2.22k op/s 2024-04-06T14:07:11.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:10 smithi090 ceph-mon[43420]: pgmap v56: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 37 MiB/s wr, 2.22k op/s 2024-04-06T14:07:11.224 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:07:11.224 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:07:11.224 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 -- 172.21.15.90:0/3096935930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e8068aa0 msgr2=0x7f73e8105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/3096935930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e8068aa0 0x7f73e8105970 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f73dc009990 tx=0x7f73dc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 -- 172.21.15.90:0/3096935930 shutdown_connections 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/3096935930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f73e81060a0 0x7f73e810dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/3096935930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e80684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/3096935930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e8068aa0 0x7f73e8105970 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 -- 172.21.15.90:0/3096935930 >> 172.21.15.90:0/3096935930 conn(0x7f73e8076d50 msgr2=0x7f73e8077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 -- 172.21.15.90:0/3096935930 shutdown_connections 2024-04-06T14:07:11.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.418+0000 7f73ee8dd700 1 -- 172.21.15.90:0/3096935930 wait complete. 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 Processor -- start 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 -- start start 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f73e81060a0 0x7f73e8199940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e819dce0 0x7f73e819e160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73e819e760 con 0x7f73e80680f0 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73e819e8d0 con 0x7f73e819dce0 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.419+0000 7f73ee8dd700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73e819ea90 con 0x7f73e81060a0 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47242/0 (socket says 172.21.15.90:47242) 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 -- 172.21.15.90:0/1489954710 learned_addr learned my addr 172.21.15.90:0/1489954710 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 -- 172.21.15.90:0/1489954710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f73e81060a0 msgr2=0x7f73e8199940 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f73e81060a0 0x7f73e8199940 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 -- 172.21.15.90:0/1489954710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e819dce0 msgr2=0x7f73e819e160 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e819dce0 0x7f73e819e160 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 -- 172.21.15.90:0/1489954710 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73d8009520 con 0x7f73e80680f0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.420+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f73d800eb30 tx=0x7f73d800eef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.421+0000 7f73e57fa700 1 -- 172.21.15.90:0/1489954710 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d8009880 con 0x7f73e80680f0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.421+0000 7f73e57fa700 1 -- 172.21.15.90:0/1489954710 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f73d80044e0 con 0x7f73e80680f0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.421+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f73dc0095f0 con 0x7f73e80680f0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.421+0000 7f73e57fa700 1 -- 172.21.15.90:0/1489954710 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d8003e70 con 0x7f73e80680f0 2024-04-06T14:07:11.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.421+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73e81a6330 con 0x7f73e80680f0 2024-04-06T14:07:11.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.423+0000 7f73e57fa700 1 -- 172.21.15.90:0/1489954710 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f73d8018430 con 0x7f73e80680f0 2024-04-06T14:07:11.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.424+0000 7f73e57fa700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 0x7f73d00a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.424+0000 7f73e57fa700 1 -- 172.21.15.90:0/1489954710 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f73d8014070 con 0x7f73e80680f0 2024-04-06T14:07:11.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.424+0000 7f73e77fe700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 0x7f73d00a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.427+0000 7f73e77fe700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 0x7f73d00a3850 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f73dc0098e0 tx=0x7f73dc005a90 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.427+0000 7f73faa91b80 1 Processor -- start 2024-04-06T14:07:11.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.427+0000 7f73faa91b80 1 -- start start 2024-04-06T14:07:11.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617c9be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617ca120 0x5644617d3920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564461721a10 con 0x5644617ca120 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56446171f3d0 con 0x5644617d3e60 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73faa91b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56446171acc0 con 0x5644617c9800 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73ece7a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617c9be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73ece7a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42114/0 (socket says 172.21.15.90:42114) 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73ece7a700 1 -- 172.21.15.90:0/938377345 learned_addr learned my addr 172.21.15.90:0/938377345 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e77fe700 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617ca120 0x5644617d3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 -- 172.21.15.90:0/938377345 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 msgr2=0x5644617d6250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 -- 172.21.15.90:0/938377345 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617ca120 msgr2=0x5644617d3920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617ca120 0x5644617d3920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73e7fff700 1 -- 172.21.15.90:0/938377345 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56446159c430 con 0x5644617c9800 2024-04-06T14:07:11.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.428+0000 7f73ece7a700 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:11.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.430+0000 7f73e7fff700 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617c9be0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f73d80cb730 tx=0x7f73d809af40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=632d136f3bfb190f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.431+0000 7f73ce7fc700 1 -- 172.21.15.90:0/938377345 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d80b2070 con 0x5644617c9800 2024-04-06T14:07:11.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.431+0000 7f73ce7fc700 1 -- 172.21.15.90:0/938377345 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f73d809b0f0 con 0x5644617c9800 2024-04-06T14:07:11.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.431+0000 7f73ce7fc700 1 -- 172.21.15.90:0/938377345 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d809b350 con 0x5644617c9800 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.431+0000 7f73faa91b80 1 -- 172.21.15.90:0/938377345 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 msgr2=0x5644617c9be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.431+0000 7f73faa91b80 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617c9be0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f73d80cb730 tx=0x7f73d809af40 comp rx=0 tx=0).stop 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 -- 172.21.15.90:0/938377345 shutdown_connections 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617d3e60 0x5644617d6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617c9be0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 --2- 172.21.15.90:0/938377345 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617ca120 0x5644617d3920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 -- 172.21.15.90:0/938377345 >> 172.21.15.90:0/938377345 conn(0x564461728720 msgr2=0x5644617c4b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 -- 172.21.15.90:0/938377345 shutdown_connections 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 -- 172.21.15.90:0/938377345 wait complete. 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 -- start start 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.432+0000 7f73faa91b80 1 Processor -- start 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617c9800 0x5644617d0bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617d3e60 0x5644617d1640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644616b6e00 con 0x5644617c9800 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644616b6aa0 con 0x5644617ca120 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56446159e720 con 0x5644617d3e60 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73faa91b80 10 client.0 _ll_register_callbacks cb 0x5644616c04f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42120/0 (socket says 172.21.15.90:42120) 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 -- 172.21.15.90:0/3608377157 learned_addr learned my addr 172.21.15.90:0/3608377157 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e7fff700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617c9800 0x5644617d0bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73ece7a700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617d3e60 0x5644617d1640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 -- 172.21.15.90:0/3608377157 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617d3e60 msgr2=0x5644617d1640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617d3e60 0x5644617d1640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 -- 172.21.15.90:0/3608377157 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617c9800 msgr2=0x5644617d0bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617c9800 0x5644617d0bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.433+0000 7f73e77fe700 1 -- 172.21.15.90:0/3608377157 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73e000e0c0 con 0x5644617ca120 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73e77fe700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f73dc015000 tx=0x7f73dc009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1a7ab7127327bfa9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73c77fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73dc01d070 con 0x5644617ca120 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f73dc004860 con 0x5644617ca120 2024-04-06T14:07:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5644616db400 con 0x5644617ca120 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f73dc004a20 con 0x5644617ca120 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73faa91b80 10 client.25430 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.434+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f73e81a6330 con 0x5644617ca120 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5644616db400 con 0x5644617ca120 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f73dc02d070 con 0x5644617ca120 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 10 client.25430 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 10 client.25430 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 20 client.25430 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644617ec250 0x5644617ee670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 4 client.25430 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:11.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564461713780 con 0x5644617ec250 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.435+0000 7f73e7fff700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644617ec250 0x5644617ee670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.438+0000 7f73c77fe700 10 client.25430 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.438+0000 7f73e7fff700 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644617ec250 0x5644617ee670 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5cc59c4af01af470 server_cookie=7413d21fb242dc90 in_seq=0 out_seq=0 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73c77fe700 1 -- 172.21.15.90:0/3608377157 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564461713780 con 0x5644617ec250 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73c77fe700 10 client.25430 handle_command_reply: tid=0 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 client.25430 shutdown 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 msgr2=0x5644617d1100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f73dc015000 tx=0x7f73dc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 shutdown_connections 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644617ec250 0x5644617ee670 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617d3e60 0x5644617d1640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617c9800 0x5644617d0bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 --2- 172.21.15.90:0/3608377157 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617d1100 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 >> 172.21.15.90:0/3608377157 conn(0x564461728720 msgr2=0x5644617c3ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73c77fe700 0 client.25430 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 shutdown_connections 2024-04-06T14:07:11.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 1 -- 172.21.15.90:0/3608377157 wait complete. 2024-04-06T14:07:11.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.439+0000 7f73faa91b80 20 client.25430 trim_cache size 0 max 16384 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.467+0000 7f73faa91b80 1 Processor -- start 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.467+0000 7f73faa91b80 1 -- start start 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617e32f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617c8f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617d1460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617d2230 con 0x5644617d3e60 2024-04-06T14:07:11.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617ca500 con 0x5644617ca120 2024-04-06T14:07:11.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73faa91b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617caa90 con 0x5644617c9800 2024-04-06T14:07:11.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617c8f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617c8f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42122/0 (socket says 172.21.15.90:42122) 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e77fe700 1 -- 172.21.15.90:0/2730182773 learned_addr learned my addr 172.21.15.90:0/2730182773 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617e32f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73ece7a700 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617d1460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 -- 172.21.15.90:0/2730182773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 msgr2=0x5644617c8f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617c8f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 -- 172.21.15.90:0/2730182773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 msgr2=0x5644617d1460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617d1460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.468+0000 7f73e7fff700 1 -- 172.21.15.90:0/2730182773 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5644617c0be0 con 0x5644617c9800 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.469+0000 7f73e7fff700 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617e32f0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f73d80cb760 tx=0x7f73d80cd040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eb99e0265832426f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.469+0000 7f73ceffd700 1 -- 172.21.15.90:0/2730182773 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d80b9040 con 0x5644617c9800 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.469+0000 7f73ceffd700 1 -- 172.21.15.90:0/2730182773 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f73d809aec0 con 0x5644617c9800 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.469+0000 7f73ceffd700 1 -- 172.21.15.90:0/2730182773 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d80b2040 con 0x5644617c9800 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 -- 172.21.15.90:0/2730182773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 msgr2=0x5644617e32f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617e32f0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f73d80cb760 tx=0x7f73d80cd040 comp rx=0 tx=0).stop 2024-04-06T14:07:11.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 -- 172.21.15.90:0/2730182773 shutdown_connections 2024-04-06T14:07:11.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617d1460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617c9800 0x5644617e32f0 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 --2- 172.21.15.90:0/2730182773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617ca120 0x5644617c8f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 -- 172.21.15.90:0/2730182773 >> 172.21.15.90:0/2730182773 conn(0x564461728720 msgr2=0x5644617c87f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.470+0000 7f73faa91b80 1 -- 172.21.15.90:0/2730182773 shutdown_connections 2024-04-06T14:07:11.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.472+0000 7f73faa91b80 1 -- 172.21.15.90:0/2730182773 wait complete. 2024-04-06T14:07:11.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.472+0000 7f73faa91b80 1 -- start start 2024-04-06T14:07:11.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 Processor -- start 2024-04-06T14:07:11.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617ca120 0x5644617e15b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617e1af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617e20c0 con 0x5644617d3e60 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617e2200 con 0x5644617c9800 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644617e23c0 con 0x5644617ca120 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.473+0000 7f73faa91b80 10 client.0 _ll_register_callbacks cb 0x5644616c04f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42138/0 (socket says 172.21.15.90:42138) 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 -- 172.21.15.90:0/1095032910 learned_addr learned my addr 172.21.15.90:0/1095032910 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73ece7a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617e1af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 -- 172.21.15.90:0/1095032910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617ca120 msgr2=0x5644617e15b0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617ca120 0x5644617e15b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 -- 172.21.15.90:0/1095032910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 msgr2=0x5644617e1af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617e1af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 -- 172.21.15.90:0/1095032910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56446159c430 con 0x5644617c9800 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73e7fff700 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f73d800eb00 tx=0x7f73d80a36a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5adaedf34b0dc6ae server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.474+0000 7f73c77fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73d80b2070 con 0x5644617c9800 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f73d809a5e0 con 0x5644617c9800 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f73d809b0f0 con 0x5644617c9800 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f73d80b9040 con 0x5644617c9800 2024-04-06T14:07:11.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 10 client.25436 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564461548be0 con 0x5644617c9800 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f73d809b0f0 con 0x5644617c9800 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f73d801fbd0 con 0x5644617c9800 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 10 client.25436 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 10 client.25436 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 20 client.25436 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644618411d0 0x5644618435c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 4 client.25436 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:07:11.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.475+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564461569d70 con 0x5644618411d0 2024-04-06T14:07:11.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.478+0000 7f73e77fe700 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644618411d0 0x5644618435c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.478+0000 7f73e77fe700 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644618411d0 0x5644618435c0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8fee06d3e84ed476 server_cookie=b45b3be9520a9430 in_seq=0 out_seq=0 2024-04-06T14:07:11.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.478+0000 7f73c77fe700 10 client.25436 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.479+0000 7f73c77fe700 1 -- 172.21.15.90:0/1095032910 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564461569d70 con 0x5644618411d0 2024-04-06T14:07:11.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.479+0000 7f73c77fe700 10 client.25436 handle_command_reply: tid=0 2024-04-06T14:07:11.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.479+0000 7f73faa91b80 1 client.25436 shutdown 2024-04-06T14:07:11.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 msgr2=0x5644617e3170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f73d800eb00 tx=0x7f73d80a36a0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 shutdown_connections 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5644617d3e60 0x5644617e1af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5644617c9800 0x5644617e3170 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5644618411d0 0x5644618435c0 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 --2- 172.21.15.90:0/1095032910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5644617ca120 0x5644617e15b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73c77fe700 0 client.25436 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 >> 172.21.15.90:0/1095032910 conn(0x564461728720 msgr2=0x5644617c9080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 shutdown_connections 2024-04-06T14:07:11.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 1 -- 172.21.15.90:0/1095032910 wait complete. 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.480+0000 7f73faa91b80 20 client.25436 trim_cache size 0 max 16384 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "5ad5dd5f-8f43-4fa0-9741-ae8eb9b89585", 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:07:11.481 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:11.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 msgr2=0x7f73d00a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 0x7f73d00a3850 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f73dc0098e0 tx=0x7f73dc005a90 comp rx=0 tx=0).stop 2024-04-06T14:07:11.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 msgr2=0x7f73e8199400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 secure :-1 s=READY pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f73d800eb30 tx=0x7f73d800eef0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 shutdown_connections 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73d00a1390 0x7f73d00a3850 secure :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f73dc0098e0 tx=0x7f73dc005a90 comp rx=0 tx=0).stop 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73e819dce0 0x7f73e819e160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f73e80680f0 0x7f73e8199400 secure :-1 s=CLOSED pgs=492 cs=0 l=1 rev1=1 crypto rx=0x7f73d800eb30 tx=0x7f73d800eef0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 --2- 172.21.15.90:0/1489954710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f73e81060a0 0x7f73e8199940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 >> 172.21.15.90:0/1489954710 conn(0x7f73e8076d50 msgr2=0x7f73e80fdf70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 shutdown_connections 2024-04-06T14:07:11.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.481+0000 7f73ee8dd700 1 -- 172.21.15.90:0/1489954710 wait complete. 2024-04-06T14:07:11.498 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:07:11.498 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:07:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.715+0000 7fb5c8927700 1 -- 172.21.15.90:0/671837560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4100820 msgr2=0x7fb5c4104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.715+0000 7fb5c8927700 1 --2- 172.21.15.90:0/671837560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4100820 0x7fb5c4104840 secure :-1 s=READY pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4009990 tx=0x7fb5b4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.716+0000 7fb5c8927700 1 -- 172.21.15.90:0/671837560 shutdown_connections 2024-04-06T14:07:11.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.716+0000 7fb5c8927700 1 --2- 172.21.15.90:0/671837560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb5c4104f70 0x7fb5c4067e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.716+0000 7fb5c8927700 1 --2- 172.21.15.90:0/671837560 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c40ffe70 0x7fb5c4100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.716+0000 7fb5c8927700 1 --2- 172.21.15.90:0/671837560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4100820 0x7fb5c4104840 secure :-1 s=CLOSED pgs=493 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4009990 tx=0x7fb5b4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.716+0000 7fb5c8927700 1 -- 172.21.15.90:0/671837560 >> 172.21.15.90:0/671837560 conn(0x7fb5c4076d50 msgr2=0x7fb5c4077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.721+0000 7fb5c8927700 1 -- 172.21.15.90:0/671837560 shutdown_connections 2024-04-06T14:07:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.725+0000 7fb5c8927700 1 -- 172.21.15.90:0/671837560 wait complete. 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.726+0000 7fb5c8927700 1 Processor -- start 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 -- start start 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb5c40ffe70 0x7fb5c410dfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 0x7fb5c410e500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 0x7fb5c410c600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c410cc50 con 0x7fb5c4104f70 2024-04-06T14:07:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c410cdc0 con 0x7fb5c40ffe70 2024-04-06T14:07:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.727+0000 7fb5c8927700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5c410cf80 con 0x7fb5c410c150 2024-04-06T14:07:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c2d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 0x7fb5c410c600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c2d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 0x7fb5c410c600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51106/0 (socket says 172.21.15.90:51106) 2024-04-06T14:07:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c2d9d700 1 -- 172.21.15.90:0/3071914684 learned_addr learned my addr 172.21.15.90:0/3071914684 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 0x7fb5c410e500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/3071914684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 msgr2=0x7fb5c410c600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 0x7fb5c410c600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/3071914684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb5c40ffe70 msgr2=0x7fb5c410dfc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb5c40ffe70 0x7fb5c410dfc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.728+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/3071914684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5b40095f0 con 0x7fb5c4104f70 2024-04-06T14:07:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.729+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 0x7fb5c410e500 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fb5c4100d50 tx=0x7fb5b400bfb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.729+0000 7fb5bb7fe700 1 -- 172.21.15.90:0/3071914684 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5b401d070 con 0x7fb5c4104f70 2024-04-06T14:07:11.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.729+0000 7fb5bb7fe700 1 -- 172.21.15.90:0/3071914684 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb5b40047e0 con 0x7fb5c4104f70 2024-04-06T14:07:11.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.729+0000 7fb5bb7fe700 1 -- 172.21.15.90:0/3071914684 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5b4005290 con 0x7fb5c4104f70 2024-04-06T14:07:11.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.730+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5c410ab50 con 0x7fb5c4104f70 2024-04-06T14:07:11.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.730+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5c41aaa60 con 0x7fb5c4104f70 2024-04-06T14:07:11.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.736+0000 7fb5d4adbb80 1 Processor -- start 2024-04-06T14:07:11.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.737+0000 7fb5bb7fe700 1 -- 172.21.15.90:0/3071914684 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb5b40053f0 con 0x7fb5c4104f70 2024-04-06T14:07:11.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 -- start start 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f38a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0f3de0 0x55560f0f61d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f041990 con 0x55560f0ea0a0 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f03f350 con 0x55560f0f3de0 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f03ac40 con 0x55560f0e9780 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5c2d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5c2d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51114/0 (socket says 172.21.15.90:51114) 2024-04-06T14:07:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.738+0000 7fb5c2d9d700 1 -- 172.21.15.90:0/369365625 learned_addr learned my addr 172.21.15.90:0/369365625 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c259c700 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5bb7fe700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 0x7fb5b00a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 0x7fb5b00a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 -- 172.21.15.90:0/369365625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0f3de0 msgr2=0x55560f0f61d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0f3de0 0x55560f0f61d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 -- 172.21.15.90:0/369365625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 msgr2=0x55560f0f38a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f38a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.739+0000 7fb5c2d9d700 1 -- 172.21.15.90:0/369365625 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55560eebc3b0 con 0x55560f0e9780 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.740+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc007b00 tx=0x7fb5bc007e10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=737fd7f64e794f79 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.740+0000 7fb5bb7fe700 1 -- 172.21.15.90:0/3071914684 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb5b40cdcb0 con 0x7fb5c4104f70 2024-04-06T14:07:11.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.740+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 0x7fb5b00a3810 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc01c000 tx=0x7fb5bc01e000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.741+0000 7fb5b8ff9700 1 -- 172.21.15.90:0/369365625 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5bc00e070 con 0x55560f0e9780 2024-04-06T14:07:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.741+0000 7fb5b8ff9700 1 -- 172.21.15.90:0/369365625 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb5bc01e070 con 0x55560f0e9780 2024-04-06T14:07:11.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.741+0000 7fb5b8ff9700 1 -- 172.21.15.90:0/369365625 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5bc0236a0 con 0x55560f0e9780 2024-04-06T14:07:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/369365625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 msgr2=0x55560f0e9b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc007b00 tx=0x7fb5bc007e10 comp rx=0 tx=0).stop 2024-04-06T14:07:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/369365625 shutdown_connections 2024-04-06T14:07:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0f3de0 0x55560f0f61d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0e9780 0x55560f0e9b60 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/369365625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f38a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/369365625 >> 172.21.15.90:0/369365625 conn(0x55560f0486a0 msgr2=0x55560f0e4a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/369365625 shutdown_connections 2024-04-06T14:07:11.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.742+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/369365625 wait complete. 2024-04-06T14:07:11.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 -- start start 2024-04-06T14:07:11.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 Processor -- start 2024-04-06T14:07:11.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0ea0a0 0x55560f0f1130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0f3de0 0x55560f0f1670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560efd6d80 con 0x55560f0f3de0 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560efd6a20 con 0x55560f0e9780 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560eebe6a0 con 0x55560f0ea0a0 2024-04-06T14:07:11.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.743+0000 7fb5d4adbb80 10 client.0 _ll_register_callbacks cb 0x55560efe0470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:11.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.744+0000 7fb5c259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.744+0000 7fb5c259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42180/0 (socket says 172.21.15.90:42180) 2024-04-06T14:07:11.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.744+0000 7fb5c259c700 1 -- 172.21.15.90:0/3381608531 learned_addr learned my addr 172.21.15.90:0/3381608531 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.744+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0f3de0 0x55560f0f1670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.744+0000 7fb5c259c700 1 -- 172.21.15.90:0/3381608531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0ea0a0 msgr2=0x55560f0f1130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5c259c700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0ea0a0 0x55560f0f1130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5c259c700 1 -- 172.21.15.90:0/3381608531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0f3de0 msgr2=0x55560f0f1670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5c259c700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0f3de0 0x55560f0f1670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5c259c700 1 -- 172.21.15.90:0/3381608531 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5bc0076e0 con 0x55560f0e9780 2024-04-06T14:07:11.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5c259c700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fb5ac0116f0 tx=0x7fb5ac011a00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3d83bdbab22c4b74 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5a9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:11.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ac004020 con 0x55560f0e9780 2024-04-06T14:07:11.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.745+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb5ac004180 con 0x55560f0e9780 2024-04-06T14:07:11.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.746+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55560effb380 con 0x55560f0e9780 2024-04-06T14:07:11.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.746+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb5ac004340 con 0x55560f0e9780 2024-04-06T14:07:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.746+0000 7fb5d4adbb80 10 client.25442 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.746+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55560ee68b60 con 0x55560f0e9780 2024-04-06T14:07:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.746+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55560effb380 con 0x55560f0e9780 2024-04-06T14:07:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb5ac00f8f0 con 0x55560f0e9780 2024-04-06T14:07:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 10 client.25442 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 10 client.25442 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 20 client.25442 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f125c10 0x55560f128030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 4 client.25442 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:11.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55560f033700 con 0x55560f125c10 2024-04-06T14:07:11.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.747+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f125c10 0x55560f128030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.752+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f125c10 0x55560f128030 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aa9497f558b949a1 server_cookie=bcb73a6acfde72b2 in_seq=0 out_seq=0 2024-04-06T14:07:11.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.752+0000 7fb5a9ffb700 10 client.25442 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/3381608531 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55560f033700 con 0x55560f125c10 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5a9ffb700 10 client.25442 handle_command_reply: tid=0 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 client.25442 shutdown 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 msgr2=0x55560f0f0bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fb5ac0116f0 tx=0x7fb5ac011a00 comp rx=0 tx=0).stop 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 shutdown_connections 2024-04-06T14:07:11.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0f3de0 0x55560f0f1670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f0bf0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f125c10 0x55560f128030 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/3381608531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0ea0a0 0x55560f0f1130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 >> 172.21.15.90:0/3381608531 conn(0x55560f0486a0 msgr2=0x55560f0e3c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.753+0000 7fb5a9ffb700 0 client.25442 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.754+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 shutdown_connections 2024-04-06T14:07:11.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.754+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/3381608531 wait complete. 2024-04-06T14:07:11.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.754+0000 7fb5d4adbb80 20 client.25442 trim_cache size 0 max 16384 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.781+0000 7fb5d4adbb80 1 Processor -- start 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.781+0000 7fb5d4adbb80 1 -- start start 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f61d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f1140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f0f1680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f0eba70 con 0x55560f0ea0a0 2024-04-06T14:07:11.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f0f3760 con 0x55560f0e9780 2024-04-06T14:07:11.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f0e6000 con 0x55560f0f3de0 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5c1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f1140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5c1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f1140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47324/0 (socket says 172.21.15.90:47324) 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/579065046 learned_addr learned my addr 172.21.15.90:0/579065046 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.782+0000 7fb5c259c700 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f61d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f0f1680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 -- 172.21.15.90:0/579065046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 msgr2=0x55560f0f1680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f0f1680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 -- 172.21.15.90:0/579065046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 msgr2=0x55560f0f1140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f1140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 -- 172.21.15.90:0/579065046 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55560f000ea0 con 0x55560f0e9780 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5c259c700 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f61d0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fb5ac0116c0 tx=0x7fb5ac0194a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e5e6fb1fa74afd8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5b97fa700 1 -- 172.21.15.90:0/579065046 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ac00f690 con 0x55560f0e9780 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5b97fa700 1 -- 172.21.15.90:0/579065046 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb5ac0055d0 con 0x55560f0e9780 2024-04-06T14:07:11.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.783+0000 7fb5b97fa700 1 -- 172.21.15.90:0/579065046 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5ac003b90 con 0x55560f0e9780 2024-04-06T14:07:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/579065046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 msgr2=0x55560f0f61d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f61d0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fb5ac0116c0 tx=0x7fb5ac0194a0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/579065046 shutdown_connections 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f0f1680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0e9780 0x55560f0f61d0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/579065046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0ea0a0 0x55560f0f1140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/579065046 >> 172.21.15.90:0/579065046 conn(0x55560f0486a0 msgr2=0x55560f0ef4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/579065046 shutdown_connections 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.784+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/579065046 wait complete. 2024-04-06T14:07:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 -- start start 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 Processor -- start 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 0x55560f109c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 0x55560f10a1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f178a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f178f80 con 0x55560f0e9780 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f1790c0 con 0x55560f0ea0a0 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55560f179250 con 0x55560f0f3de0 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5d4adbb80 10 client.0 _ll_register_callbacks cb 0x55560efe0470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5c259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 0x55560f109c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5c1d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 0x55560f10a1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5c259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 0x55560f109c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47328/0 (socket says 172.21.15.90:47328) 2024-04-06T14:07:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5c259c700 1 -- 172.21.15.90:0/308166250 learned_addr learned my addr 172.21.15.90:0/308166250 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.785+0000 7fb5c2d9d700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f178a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/308166250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 msgr2=0x55560f178a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f178a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/308166250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 msgr2=0x55560f109c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 0x55560f109c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 -- 172.21.15.90:0/308166250 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55560ee68b60 con 0x55560f0ea0a0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5c1d9b700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 0x55560f10a1b0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4097360 tx=0x7fb5b40971e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6d485b3c1d9bcbaf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5a9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5b409ec90 con 0x55560f0ea0a0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb5b40a3250 con 0x55560f0ea0a0 2024-04-06T14:07:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.786+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb5bc01e070 con 0x55560f0ea0a0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb5b40163a0 con 0x55560f0ea0a0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 10 client.25448 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55560eec0cd0 con 0x55560f0ea0a0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb5bc01e070 con 0x55560f0ea0a0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb5b409a350 con 0x55560f0ea0a0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 10 client.25448 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 10 client.25448 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 20 client.25448 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f17a740 0x55560f17cb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 4 client.25448 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.787+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55560ee89cf0 con 0x55560f17a740 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.788+0000 7fb5c259c700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f17a740 0x55560f17cb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.788+0000 7fb5a9ffb700 10 client.25448 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.788+0000 7fb5c259c700 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f17a740 0x55560f17cb60 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a16e97fe41c19c33 server_cookie=468454b4999a289b in_seq=0 out_seq=0 2024-04-06T14:07:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.789+0000 7fb5a9ffb700 1 -- 172.21.15.90:0/308166250 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55560ee89cf0 con 0x55560f17a740 2024-04-06T14:07:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.789+0000 7fb5a9ffb700 10 client.25448 handle_command_reply: tid=0 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.789+0000 7fb5d4adbb80 1 client.25448 shutdown 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 msgr2=0x55560f10a1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 0x55560f10a1b0 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4097360 tx=0x7fb5b40971e0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 shutdown_connections 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55560f17a740 0x55560f17cb60 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55560f0f3de0 0x55560f178a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5a9ffb700 0 client.25448 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55560f0e9780 0x55560f109c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 --2- 172.21.15.90:0/308166250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55560f0ea0a0 0x55560f10a1b0 secure :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fb5b4097360 tx=0x7fb5b40971e0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 >> 172.21.15.90:0/308166250 conn(0x55560f0486a0 msgr2=0x55560f0f2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 shutdown_connections 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 1 -- 172.21.15.90:0/308166250 wait complete. 2024-04-06T14:07:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.790+0000 7fb5d4adbb80 20 client.25448 trim_cache size 0 max 16384 2024-04-06T14:07:11.794 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:11.794 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:07:11.794 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: "5ad5dd5f-8f43-4fa0-9741-ae8eb9b89585": { 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "5ad5dd5f-8f43-4fa0-9741-ae8eb9b89585", 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:07:11.795 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 msgr2=0x7fb5b00a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 0x7fb5b00a3810 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fb5bc01c000 tx=0x7fb5bc01e000 comp rx=0 tx=0).stop 2024-04-06T14:07:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 msgr2=0x7fb5c410e500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 0x7fb5c410e500 secure :-1 s=READY pgs=494 cs=0 l=1 rev1=1 crypto rx=0x7fb5c4100d50 tx=0x7fb5b400bfb0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 shutdown_connections 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb5b00a1350 0x7fb5b00a3810 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb5c410c150 0x7fb5c410c600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb5c40ffe70 0x7fb5c410dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 --2- 172.21.15.90:0/3071914684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5c4104f70 0x7fb5c410e500 unknown :-1 s=CLOSED pgs=494 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 >> 172.21.15.90:0/3071914684 conn(0x7fb5c4076d50 msgr2=0x7fb5c4107640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 shutdown_connections 2024-04-06T14:07:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:11.791+0000 7fb5c8927700 1 -- 172.21.15.90:0/3071914684 wait complete. 2024-04-06T14:07:11.807 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:11 smithi090 ceph-mon[43420]: pgmap v57: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:11.808 INFO:tasks.cephfs.filesystem:scrub status for tag:5ad5dd5f-8f43-4fa0-9741-ae8eb9b89585 - {'path': '/', 'tag': '5ad5dd5f-8f43-4fa0-9741-ae8eb9b89585', 'options': 'recursive,force'} 2024-04-06T14:07:11.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:11 smithi105 ceph-mon[46969]: pgmap v57: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:12.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:11 smithi134 ceph-mon[47549]: pgmap v57: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:12 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:07:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:12 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:12 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:07:12.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:12 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 85.0 s, 3446.2 tps, lat 29.020 ms stddev 32.784 2024-04-06T14:07:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:12 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:07:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:12 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:12 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:07:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:12 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:12 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:07:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:12 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:12 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:07:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:12 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:13.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:13 smithi105 ceph-mon[46969]: pgmap v58: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:14.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:13 smithi090 ceph-mon[43420]: pgmap v58: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:14.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:13 smithi134 ceph-mon[47549]: pgmap v58: 97 pgs: 97 active+clean; 2.8 GiB data, 3.4 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 25 MiB/s wr, 1.73k op/s 2024-04-06T14:07:15.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:15 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:07:15.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:15 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:07:15.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:15 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:07:16.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:16 smithi090 ceph-mon[43420]: pgmap v59: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 37 MiB/s wr, 2.21k op/s 2024-04-06T14:07:16.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:16 smithi134 ceph-mon[47549]: pgmap v59: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 37 MiB/s wr, 2.21k op/s 2024-04-06T14:07:16.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:16 smithi105 ceph-mon[46969]: pgmap v59: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 37 MiB/s wr, 2.21k op/s 2024-04-06T14:07:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 90.0 s, 3901.3 tps, lat 25.595 ms stddev 27.137 2024-04-06T14:07:18.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:18 smithi134 ceph-mon[47549]: pgmap v60: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 835 op/s 2024-04-06T14:07:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:18 smithi105 ceph-mon[46969]: pgmap v60: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 835 op/s 2024-04-06T14:07:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:18 smithi090 ceph-mon[43420]: pgmap v60: 97 pgs: 97 active+clean; 2.9 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 18 MiB/s wr, 835 op/s 2024-04-06T14:07:19.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:19 smithi105 ceph-mon[46969]: pgmap v61: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 32 MiB/s wr, 1.31k op/s 2024-04-06T14:07:20.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:19 smithi090 ceph-mon[43420]: pgmap v61: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 32 MiB/s wr, 1.31k op/s 2024-04-06T14:07:20.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:19 smithi134 ceph-mon[47549]: pgmap v61: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 32 MiB/s wr, 1.31k op/s 2024-04-06T14:07:21.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:21 smithi105 ceph-mon[46969]: pgmap v62: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:22.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:21 smithi090 ceph-mon[43420]: pgmap v62: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:22.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:21 smithi134 ceph-mon[47549]: pgmap v62: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:23.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 95.0 s, 3290.5 tps, lat 30.127 ms stddev 44.865 2024-04-06T14:07:23.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:23 smithi105 ceph-mon[46969]: pgmap v63: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:24.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:23 smithi090 ceph-mon[43420]: pgmap v63: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:24.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:23 smithi134 ceph-mon[47549]: pgmap v63: 97 pgs: 97 active+clean; 3.1 GiB data, 3.9 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 25 MiB/s wr, 955 op/s 2024-04-06T14:07:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:25 smithi105 ceph-mon[46969]: pgmap v64: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 56 MiB/s wr, 2.47k op/s 2024-04-06T14:07:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:25 smithi090 ceph-mon[43420]: pgmap v64: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 56 MiB/s wr, 2.47k op/s 2024-04-06T14:07:26.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:25 smithi134 ceph-mon[47549]: pgmap v64: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 56 MiB/s wr, 2.47k op/s 2024-04-06T14:07:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 100.0 s, 4499.7 tps, lat 22.448 ms stddev 25.729 2024-04-06T14:07:28.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:28 smithi134 ceph-mon[47549]: pgmap v65: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 45 MiB/s wr, 1.98k op/s 2024-04-06T14:07:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:28 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:07:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:28 smithi105 ceph-mon[46969]: pgmap v65: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 45 MiB/s wr, 1.98k op/s 2024-04-06T14:07:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:28 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:07:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:28 smithi090 ceph-mon[43420]: pgmap v65: 97 pgs: 97 active+clean; 3.4 GiB data, 4.2 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 45 MiB/s wr, 1.98k op/s 2024-04-06T14:07:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:28 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:07:29.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:29 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:07:29.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:29 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:07:29.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:29 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:07:29.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:29 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:07:29.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:29 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:07:29.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:29 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:07:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:29 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:07:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:29 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:07:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:29 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:07:30.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:30 smithi134 ceph-mon[47549]: pgmap v66: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 53 MiB/s wr, 2.34k op/s 2024-04-06T14:07:30.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:30 smithi105 ceph-mon[46969]: pgmap v66: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 53 MiB/s wr, 2.34k op/s 2024-04-06T14:07:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:30 smithi090 ceph-mon[43420]: pgmap v66: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 53 MiB/s wr, 2.34k op/s 2024-04-06T14:07:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:31 smithi105 ceph-mon[46969]: pgmap v67: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:31 smithi090 ceph-mon[43420]: pgmap v67: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:32.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:31 smithi134 ceph-mon[47549]: pgmap v67: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:33.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 105.0 s, 4031.7 tps, lat 24.842 ms stddev 29.885 2024-04-06T14:07:33.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:33 smithi105 ceph-mon[46969]: pgmap v68: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:34.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:33 smithi090 ceph-mon[43420]: pgmap v68: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:34.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:33 smithi134 ceph-mon[47549]: pgmap v68: 97 pgs: 97 active+clean; 3.5 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 39 MiB/s wr, 1.87k op/s 2024-04-06T14:07:35.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:35 smithi105 ceph-mon[46969]: pgmap v69: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:07:36.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:35 smithi090 ceph-mon[43420]: pgmap v69: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:07:36.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:35 smithi134 ceph-mon[47549]: pgmap v69: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:07:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 110.0 s, 3836.5 tps, lat 25.914 ms stddev 27.967 2024-04-06T14:07:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:38 smithi134 ceph-mon[47549]: pgmap v70: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 15 MiB/s wr, 701 op/s 2024-04-06T14:07:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:38 smithi105 ceph-mon[46969]: pgmap v70: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 15 MiB/s wr, 701 op/s 2024-04-06T14:07:39.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:38 smithi090 ceph-mon[43420]: pgmap v70: 97 pgs: 97 active+clean; 3.5 GiB data, 4.4 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 15 MiB/s wr, 701 op/s 2024-04-06T14:07:39.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:39 smithi105 ceph-mon[46969]: pgmap v71: 97 pgs: 97 active+clean; 3.6 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 23 MiB/s wr, 1.12k op/s 2024-04-06T14:07:40.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:39 smithi090 ceph-mon[43420]: pgmap v71: 97 pgs: 97 active+clean; 3.6 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 23 MiB/s wr, 1.12k op/s 2024-04-06T14:07:40.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:39 smithi134 ceph-mon[47549]: pgmap v71: 97 pgs: 97 active+clean; 3.6 GiB data, 4.5 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 23 MiB/s wr, 1.12k op/s 2024-04-06T14:07:41.810 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:07:41.810 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:07:41.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:41 smithi105 ceph-mon[46969]: pgmap v72: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:42.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.022+0000 7ff3e774e700 1 -- 172.21.15.90:0/2319583165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 msgr2=0x7ff3e010caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.022+0000 7ff3e774e700 1 --2- 172.21.15.90:0/2319583165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e010caf0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7ff3d801c0d0 tx=0x7ff3d801c3e0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 -- 172.21.15.90:0/2319583165 shutdown_connections 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 --2- 172.21.15.90:0/2319583165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e010caf0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 --2- 172.21.15.90:0/2319583165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff3e00ffe70 0x7ff3e0100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 --2- 172.21.15.90:0/2319583165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff3e0100820 0x7ff3e0104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 -- 172.21.15.90:0/2319583165 >> 172.21.15.90:0/2319583165 conn(0x7ff3e0076d50 msgr2=0x7ff3e0077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 -- 172.21.15.90:0/2319583165 shutdown_connections 2024-04-06T14:07:42.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.023+0000 7ff3e774e700 1 -- 172.21.15.90:0/2319583165 wait complete. 2024-04-06T14:07:42.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 Processor -- start 2024-04-06T14:07:42.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 -- start start 2024-04-06T14:07:42.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff3e00ffe70 0x7ff3e006d400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff3e019eaf0 0x7ff3e01a0ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff3e01a14e0 con 0x7ff3e00ffe70 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff3e01a1650 con 0x7ff3e0104f70 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e774e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff3e01a1810 con 0x7ff3e019eaf0 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e4ce9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e4ce9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34950/0 (socket says 172.21.15.90:34950) 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.024+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/985185334 learned_addr learned my addr 172.21.15.90:0/985185334 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/985185334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff3e019eaf0 msgr2=0x7ff3e01a0ee0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff3e019eaf0 0x7ff3e01a0ee0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff3e00ffe70 msgr2=0x7ff3e006d400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff3e00ffe70 0x7ff3e006d400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/985185334 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff3d801ad10 con 0x7ff3e0104f70 2024-04-06T14:07:42.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7ff3d000d980 tx=0x7ff3d000dd40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3d67fc700 1 -- 172.21.15.90:0/985185334 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d0009ac0 con 0x7ff3e0104f70 2024-04-06T14:07:42.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3d67fc700 1 -- 172.21.15.90:0/985185334 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3d000de60 con 0x7ff3e0104f70 2024-04-06T14:07:42.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.025+0000 7ff3d67fc700 1 -- 172.21.15.90:0/985185334 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d0010ea0 con 0x7ff3e0104f70 2024-04-06T14:07:42.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.026+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff3e01a1af0 con 0x7ff3e0104f70 2024-04-06T14:07:42.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.026+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff3e01a1f80 con 0x7ff3e0104f70 2024-04-06T14:07:42.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.028+0000 7ff3d67fc700 1 -- 172.21.15.90:0/985185334 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff3d0010460 con 0x7ff3e0104f70 2024-04-06T14:07:42.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.029+0000 7ff3d67fc700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 0x7ff3cc0a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.030+0000 7ff3d67fc700 1 -- 172.21.15.90:0/985185334 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3d00cc790 con 0x7ff3e0104f70 2024-04-06T14:07:42.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.030+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 0x7ff3cc0a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 0x7ff3cc0a3810 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc005950 tx=0x7ff3dc0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 Processor -- start 2024-04-06T14:07:42.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 -- start start 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3b6b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3c31d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a30e990 con 0x56427a3c0de0 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a30c350 con 0x56427a3b6780 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.031+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a307c40 con 0x56427a3b70a0 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34856/0 (socket says 172.21.15.90:34856) 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/2469129471 learned_addr learned my addr 172.21.15.90:0/2469129471 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3c31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3b6b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/2469129471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 msgr2=0x56427a3b6b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3b6b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/2469129471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 msgr2=0x56427a3c31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3c31d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/2469129471 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56427a1893b0 con 0x56427a3b70a0 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7ff3d00a1800 tx=0x7ff3d0099520 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a4b8a90b1be43455 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3cb7fe700 1 -- 172.21.15.90:0/2469129471 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d00b7ea0 con 0x56427a3b70a0 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3cb7fe700 1 -- 172.21.15.90:0/2469129471 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3d00a1d30 con 0x56427a3b70a0 2024-04-06T14:07:42.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.032+0000 7ff3cb7fe700 1 -- 172.21.15.90:0/2469129471 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d0099d50 con 0x56427a3b70a0 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 -- 172.21.15.90:0/2469129471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 msgr2=0x56427a3c08a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7ff3d00a1800 tx=0x7ff3d0099520 comp rx=0 tx=0).stop 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 -- 172.21.15.90:0/2469129471 shutdown_connections 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3c31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3b6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/2469129471 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3c08a0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.033+0000 7ff3f3902b80 1 -- 172.21.15.90:0/2469129471 >> 172.21.15.90:0/2469129471 conn(0x56427a3156a0 msgr2=0x56427a3b1a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- 172.21.15.90:0/2469129471 shutdown_connections 2024-04-06T14:07:42.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- 172.21.15.90:0/2469129471 wait complete. 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- start start 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 Processor -- start 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3bdb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3be2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a2a3d80 con 0x56427a3c0de0 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a2a3a20 con 0x56427a3b6780 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.034+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a18b6a0 con 0x56427a3b70a0 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3f3902b80 10 client.0 _ll_register_callbacks cb 0x56427a2ad470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34960/0 (socket says 172.21.15.90:34960) 2024-04-06T14:07:42.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 -- 172.21.15.90:0/353365023 learned_addr learned my addr 172.21.15.90:0/353365023 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3bdb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3be2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 -- 172.21.15.90:0/353365023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 msgr2=0x56427a3bdb80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3bdb80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 -- 172.21.15.90:0/353365023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 msgr2=0x56427a3be2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3be2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 -- 172.21.15.90:0/353365023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff3e01a1f80 con 0x56427a3b6780 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3be2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc00df80 tx=0x7ff3dc004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7fb82f992b9b1a20 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3c8ff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3dc010e80 con 0x56427a3b6780 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.035+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3dc026400 con 0x56427a3b6780 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56427a2c8380 con 0x56427a3b6780 2024-04-06T14:07:42.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff3dc0265c0 con 0x56427a3b6780 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 10 client.25460 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56427a135b60 con 0x56427a3b6780 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56427a2c8380 con 0x56427a3b6780 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff3dc02d070 con 0x56427a3b6780 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 10 client.25460 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 10 client.25460 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 20 client.25460 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a3c6900 0x56427a3c8d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 4 client.25460 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56427a300700 con 0x56427a3c6900 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.036+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a3c6900 0x56427a3c8d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.037+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a3c6900 0x56427a3c8d20 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6ea610e3f6a747a6 server_cookie=a2d45ca13ccbddc0 in_seq=0 out_seq=0 2024-04-06T14:07:42.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3c8ff9700 10 client.25460 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:42.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/353365023 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56427a300700 con 0x56427a3c6900 2024-04-06T14:07:42.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3c8ff9700 10 client.25460 handle_command_reply: tid=0 2024-04-06T14:07:42.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:41 smithi090 ceph-mon[43420]: pgmap v72: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:42.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 client.25460 shutdown 2024-04-06T14:07:42.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 msgr2=0x56427a3c0890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc00df80 tx=0x7ff3dc004750 comp rx=0 tx=0).stop 2024-04-06T14:07:42.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 shutdown_connections 2024-04-06T14:07:42.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3c0de0 0x56427a3be2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b6780 0x56427a3c0890 secure :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc00df80 tx=0x7ff3dc004750 comp rx=0 tx=0).stop 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a3c6900 0x56427a3c8d20 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/353365023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3b70a0 0x56427a3bdb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 >> 172.21.15.90:0/353365023 conn(0x56427a3156a0 msgr2=0x56427a3b0c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.039+0000 7ff3c8ff9700 0 client.25460 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.040+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 shutdown_connections 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.040+0000 7ff3f3902b80 1 -- 172.21.15.90:0/353365023 wait complete. 2024-04-06T14:07:42.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.040+0000 7ff3f3902b80 20 client.25460 trim_cache size 0 max 16384 2024-04-06T14:07:42.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.061+0000 7ff3f3902b80 1 Processor -- start 2024-04-06T14:07:42.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 -- start start 2024-04-06T14:07:42.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 0x56427a3bdb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d64d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 0x56427a3b1800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a3b3220 con 0x56427a3b13a0 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a3bc0d0 con 0x56427a3b70a0 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a3b9770 con 0x56427a3c0de0 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 0x56427a3bdb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d64d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 0x56427a3bdb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34972/0 (socket says 172.21.15.90:34972) 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e54ea700 1 -- 172.21.15.90:0/1662421349 learned_addr learned my addr 172.21.15.90:0/1662421349 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 0x56427a3b1800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/1662421349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 msgr2=0x56427a3bdb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 0x56427a3bdb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/1662421349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 msgr2=0x56427a3b1800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 0x56427a3b1800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e4ce9700 1 -- 172.21.15.90:0/1662421349 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56427a2cdea0 con 0x56427a3c0de0 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.062+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 0x56427a3b1800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.063+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d64d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7ff3d00995b0 tx=0x7ff3d00999f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4fc2bdacb55b71c6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.063+0000 7ff3cbfff700 1 -- 172.21.15.90:0/1662421349 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d00b7ea0 con 0x56427a3c0de0 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.063+0000 7ff3cbfff700 1 -- 172.21.15.90:0/1662421349 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3d00b2260 con 0x56427a3c0de0 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.063+0000 7ff3cbfff700 1 -- 172.21.15.90:0/1662421349 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3d0099000 con 0x56427a3c0de0 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 -- 172.21.15.90:0/1662421349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 msgr2=0x56427a3d64d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d64d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7ff3d00995b0 tx=0x7ff3d00999f0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 -- 172.21.15.90:0/1662421349 shutdown_connections 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b13a0 0x56427a3b1800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b70a0 0x56427a3bdb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/1662421349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d64d0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.064+0000 7ff3f3902b80 1 -- 172.21.15.90:0/1662421349 >> 172.21.15.90:0/1662421349 conn(0x56427a3156a0 msgr2=0x56427a3bd440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.065+0000 7ff3f3902b80 1 -- 172.21.15.90:0/1662421349 shutdown_connections 2024-04-06T14:07:42.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.065+0000 7ff3f3902b80 1 -- 172.21.15.90:0/1662421349 wait complete. 2024-04-06T14:07:42.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.065+0000 7ff3f3902b80 1 -- start start 2024-04-06T14:07:42.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.065+0000 7ff3f3902b80 1 Processor -- start 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 0x56427a3d49f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d4f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a445df0 con 0x56427a3b70a0 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a445f60 con 0x56427a3b13a0 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56427a446120 con 0x56427a3c0de0 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3f3902b80 10 client.0 _ll_register_callbacks cb 0x56427a2ad470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34976/0 (socket says 172.21.15.90:34976) 2024-04-06T14:07:42.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 -- 172.21.15.90:0/3224263834 learned_addr learned my addr 172.21.15.90:0/3224263834 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e5ceb700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 0x56427a3d49f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 -- 172.21.15.90:0/3224263834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 msgr2=0x56427a3d4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d4f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 -- 172.21.15.90:0/3224263834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 msgr2=0x56427a3d49f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 0x56427a3d49f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e54ea700 1 -- 172.21.15.90:0/3224263834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff3e010d360 con 0x56427a3b13a0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.066+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 0x56427a3d49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3e54ea700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc00df80 tx=0x7ff3dc010db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2dc880a2d5c94a61 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3c8ff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3dc02f830 con 0x56427a3b13a0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3dc01ea90 con 0x56427a3b13a0 2024-04-06T14:07:42.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff3d00a1d30 con 0x56427a3b13a0 2024-04-06T14:07:42.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff3dc02f990 con 0x56427a3b13a0 2024-04-06T14:07:42.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3f3902b80 10 client.25466 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:42.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.067+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56427a1893b0 con 0x56427a3b13a0 2024-04-06T14:07:42.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.068+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3d00a1d30 con 0x56427a3b13a0 2024-04-06T14:07:42.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.068+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff3dc02d050 con 0x56427a3b13a0 2024-04-06T14:07:42.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 10 client.25466 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:42.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 10 client.25466 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:42.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 20 client.25466 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a4477f0 0x56427a449c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 4 client.25466 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56427a156cf0 con 0x56427a4477f0 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a4477f0 0x56427a449c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3c8ff9700 10 client.25466 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3e4ce9700 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a4477f0 0x56427a449c10 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=960ffc7d8ed5170 server_cookie=64660ebbbf0be3ed in_seq=0 out_seq=0 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3c8ff9700 1 -- 172.21.15.90:0/3224263834 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x56427a156cf0 con 0x56427a4477f0 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3c8ff9700 10 client.25466 handle_command_reply: tid=0 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 1 client.25466 shutdown 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 msgr2=0x56427a3d44b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.069+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc00df80 tx=0x7ff3dc010db0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 shutdown_connections 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56427a4477f0 0x56427a449c10 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56427a3c0de0 0x56427a3d4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56427a3b13a0 0x56427a3d44b0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3c8ff9700 0 client.25466 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 --2- 172.21.15.90:0/3224263834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56427a3b70a0 0x56427a3d49f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 >> 172.21.15.90:0/3224263834 conn(0x56427a3156a0 msgr2=0x56427a3dcc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 shutdown_connections 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 1 -- 172.21.15.90:0/3224263834 wait complete. 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3f3902b80 20 client.25466 trim_cache size 0 max 16384 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:07:42.073 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:07:42.074 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:42.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 msgr2=0x7ff3cc0a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 0x7ff3cc0a3810 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc005950 tx=0x7ff3dc0058e0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 msgr2=0x7ff3e006d940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.070+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7ff3d000d980 tx=0x7ff3d000dd40 comp rx=0 tx=0).stop 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 shutdown_connections 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3cc0a1350 0x7ff3cc0a3810 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff3e019eaf0 0x7ff3e01a0ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff3e00ffe70 0x7ff3e006d400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 --2- 172.21.15.90:0/985185334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e0104f70 0x7ff3e006d940 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.071+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 >> 172.21.15.90:0/985185334 conn(0x7ff3e0076d50 msgr2=0x7ff3e00fdd00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.073+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 shutdown_connections 2024-04-06T14:07:42.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.073+0000 7ff3e774e700 1 -- 172.21.15.90:0/985185334 wait complete. 2024-04-06T14:07:42.091 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:07:42.092 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:07:42.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:41 smithi134 ceph-mon[47549]: pgmap v72: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 -- 172.21.15.90:0/1317088487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea600ff3b0 msgr2=0x7fea600ff790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 --2- 172.21.15.90:0/1317088487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea600ff3b0 0x7fea600ff790 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fea5c0098f0 tx=0x7fea5c009c00 comp rx=0 tx=0).stop 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 -- 172.21.15.90:0/1317088487 shutdown_connections 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 --2- 172.21.15.90:0/1317088487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea60105f70 0x7fea6010db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 --2- 172.21.15.90:0/1317088487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea600ff3b0 0x7fea600ff790 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 --2- 172.21.15.90:0/1317088487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea600ffd60 0x7fea60105890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 -- 172.21.15.90:0/1317088487 >> 172.21.15.90:0/1317088487 conn(0x7fea600760a0 msgr2=0x7fea600fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 -- 172.21.15.90:0/1317088487 shutdown_connections 2024-04-06T14:07:42.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.266+0000 7fea6758e700 1 -- 172.21.15.90:0/1317088487 wait complete. 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 Processor -- start 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 -- start start 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 0x7fea60100860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea60101fe0 0x7fea60100de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea601a9d80 con 0x7fea600ff3b0 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea601a9ef0 con 0x7fea60105f70 2024-04-06T14:07:42.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.267+0000 7fea6758e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea601aa0b0 con 0x7fea60101fe0 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34302/0 (socket says 172.21.15.90:34302) 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 -- 172.21.15.90:0/1547025046 learned_addr learned my addr 172.21.15.90:0/1547025046 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea64b29700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 0x7fea60100860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea65b2b700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea60101fe0 0x7fea60100de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 -- 172.21.15.90:0/1547025046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea60101fe0 msgr2=0x7fea60100de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea60101fe0 0x7fea60100de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 -- 172.21.15.90:0/1547025046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 msgr2=0x7fea60100860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 0x7fea60100860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 -- 172.21.15.90:0/1547025046 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea50009520 con 0x7fea600ff3b0 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea64b29700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 0x7fea60100860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:42.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.268+0000 7fea6532a700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7fea5c009230 tx=0x7fea5c00fa00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.269+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea5c01c070 con 0x7fea600ff3b0 2024-04-06T14:07:42.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.269+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea5c00baa0 con 0x7fea600ff3b0 2024-04-06T14:07:42.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.269+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea5c004930 con 0x7fea600ff3b0 2024-04-06T14:07:42.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.269+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea5c009520 con 0x7fea600ff3b0 2024-04-06T14:07:42.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.269+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea601aa570 con 0x7fea600ff3b0 2024-04-06T14:07:42.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.271+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea60109200 con 0x7fea600ff3b0 2024-04-06T14:07:42.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.272+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fea5c020070 con 0x7fea600ff3b0 2024-04-06T14:07:42.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.272+0000 7fea567fc700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 0x7fea4c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.273+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fea5c0cd7e0 con 0x7fea600ff3b0 2024-04-06T14:07:42.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.274+0000 7fea64b29700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 0x7fea4c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.274+0000 7fea64b29700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 0x7fea4c0a37d0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fea50009fd0 tx=0x7fea50009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.280+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fea5c096cd0 con 0x7fea600ff3b0 2024-04-06T14:07:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.670+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fea60102460 con 0x7fea600ff3b0 2024-04-06T14:07:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.671+0000 7fea567fc700 1 -- 172.21.15.90:0/1547025046 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fea5c09a2f0 con 0x7fea600ff3b0 2024-04-06T14:07:42.671 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:07:42.672 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 msgr2=0x7fea4c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 0x7fea4c0a37d0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fea50009fd0 tx=0x7fea50009260 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 msgr2=0x7fea60100320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 secure :-1 s=READY pgs=495 cs=0 l=1 rev1=1 crypto rx=0x7fea5c009230 tx=0x7fea5c00fa00 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 shutdown_connections 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea4c0a1310 0x7fea4c0a37d0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea60101fe0 0x7fea60100de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea600ff3b0 0x7fea60100320 unknown :-1 s=CLOSED pgs=495 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 --2- 172.21.15.90:0/1547025046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea60105f70 0x7fea60100860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 >> 172.21.15.90:0/1547025046 conn(0x7fea600760a0 msgr2=0x7fea601042a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 shutdown_connections 2024-04-06T14:07:42.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.674+0000 7fea6758e700 1 -- 172.21.15.90:0/1547025046 wait complete. 2024-04-06T14:07:42.676 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:07:42.692 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:42.692 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 -- 172.21.15.90:0/1513517987 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 msgr2=0x7f0768106180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 --2- 172.21.15.90:0/1513517987 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f0768106180 secure :-1 s=READY pgs=496 cs=0 l=1 rev1=1 crypto rx=0x7f075c0099c0 tx=0x7f075c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 -- 172.21.15.90:0/1513517987 shutdown_connections 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 --2- 172.21.15.90:0/1513517987 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f07681008c0 0x7f076810cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 --2- 172.21.15.90:0/1513517987 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f0768106180 unknown :-1 s=CLOSED pgs=496 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 --2- 172.21.15.90:0/1513517987 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07680fff00 0x7f0768100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 -- 172.21.15.90:0/1513517987 >> 172.21.15.90:0/1513517987 conn(0x7f07680769b0 msgr2=0x7f0768076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.883+0000 7f076f389700 1 -- 172.21.15.90:0/1513517987 shutdown_connections 2024-04-06T14:07:42.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 -- 172.21.15.90:0/1513517987 wait complete. 2024-04-06T14:07:42.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 Processor -- start 2024-04-06T14:07:42.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 -- start start 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f07680fff00 0x7f0768196040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 0x7f0768196580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f076819af60 con 0x7f0768105da0 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f076819b0d0 con 0x7f07681008c0 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.884+0000 7f076f389700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f076819b290 con 0x7f07680fff00 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.885+0000 7f076d926700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.885+0000 7f076d926700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34328/0 (socket says 172.21.15.90:34328) 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.885+0000 7f076d926700 1 -- 172.21.15.90:0/4039488209 learned_addr learned my addr 172.21.15.90:0/4039488209 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.885+0000 7f076c924700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 0x7f0768196580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 -- 172.21.15.90:0/4039488209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f07680fff00 msgr2=0x7f0768196040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f07680fff00 0x7f0768196040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 -- 172.21.15.90:0/4039488209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 msgr2=0x7f0768196580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 0x7f0768196580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 -- 172.21.15.90:0/4039488209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f075c0095f0 con 0x7f0768105da0 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076c924700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 0x7f0768196580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:42.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.886+0000 7f076d926700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f076000e580 tx=0x7f076000e940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.887+0000 7f075a7fc700 1 -- 172.21.15.90:0/4039488209 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0760004bb0 con 0x7f0768105da0 2024-04-06T14:07:42.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.887+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f076819b570 con 0x7f0768105da0 2024-04-06T14:07:42.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.887+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0768069120 con 0x7f0768105da0 2024-04-06T14:07:42.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.887+0000 7f075a7fc700 1 -- 172.21.15.90:0/4039488209 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0760004d10 con 0x7f0768105da0 2024-04-06T14:07:42.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.887+0000 7f075a7fc700 1 -- 172.21.15.90:0/4039488209 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07600048b0 con 0x7f0768105da0 2024-04-06T14:07:42.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.890+0000 7f075a7fc700 1 -- 172.21.15.90:0/4039488209 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0760007500 con 0x7f0768105da0 2024-04-06T14:07:42.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.890+0000 7f075a7fc700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 0x7f07540a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.890+0000 7f075a7fc700 1 -- 172.21.15.90:0/4039488209 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0760016070 con 0x7f0768105da0 2024-04-06T14:07:42.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f076d125700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 0x7f07540a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 Processor -- start 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 -- start start 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece84c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece79a990 con 0x55eece84cde0 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece798350 con 0x55eece8430a0 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.891+0000 7f077b53db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece793c40 con 0x55eece842780 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34344/0 (socket says 172.21.15.90:34344) 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 -- 172.21.15.90:0/2485571470 learned_addr learned my addr 172.21.15.90:0/2485571470 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076c924700 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece84c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d125700 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d125700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 0x7f07540a3850 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f075c0098e0 tx=0x7f075c003680 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 -- 172.21.15.90:0/2485571470 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece842780 msgr2=0x55eece842b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 -- 172.21.15.90:0/2485571470 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 msgr2=0x55eece84c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece84c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 -- 172.21.15.90:0/2485571470 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eece6153b0 con 0x55eece84cde0 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f076d926700 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f0760004420 tx=0x7f076009a930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=da26b3d75d3ab30d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.892+0000 7f07537fe700 1 -- 172.21.15.90:0/2485571470 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f076009adf0 con 0x55eece84cde0 2024-04-06T14:07:42.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.893+0000 7f07537fe700 1 -- 172.21.15.90:0/2485571470 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f076009ec20 con 0x55eece84cde0 2024-04-06T14:07:42.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.893+0000 7f07537fe700 1 -- 172.21.15.90:0/2485571470 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f076009a540 con 0x55eece84cde0 2024-04-06T14:07:42.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 -- 172.21.15.90:0/2485571470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 msgr2=0x55eece84f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 secure :-1 s=READY pgs=498 cs=0 l=1 rev1=1 crypto rx=0x7f0760004420 tx=0x7f076009a930 comp rx=0 tx=0).stop 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 -- 172.21.15.90:0/2485571470 shutdown_connections 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece84cde0 0x55eece84f1d0 unknown :-1 s=CLOSED pgs=498 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 --2- 172.21.15.90:0/2485571470 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece84c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 -- 172.21.15.90:0/2485571470 >> 172.21.15.90:0/2485571470 conn(0x55eece7a16a0 msgr2=0x55eece83da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 -- 172.21.15.90:0/2485571470 shutdown_connections 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.894+0000 7f077b53db80 1 -- 172.21.15.90:0/2485571470 wait complete. 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.895+0000 7f077b53db80 1 -- start start 2024-04-06T14:07:42.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.895+0000 7f077b53db80 1 Processor -- start 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece84c7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece849b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece84a130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece72fd80 con 0x55eece842780 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece72fa20 con 0x55eece8430a0 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece6176a0 con 0x55eece84cde0 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f077b53db80 10 client.0 _ll_register_callbacks cb 0x55eece739470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076c924700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece849b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076c924700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece849b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35030/0 (socket says 172.21.15.90:35030) 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076c924700 1 -- 172.21.15.90:0/2493907977 learned_addr learned my addr 172.21.15.90:0/2493907977 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece84a130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d125700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece84c7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 -- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 msgr2=0x55eece849b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece849b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 -- 172.21.15.90:0/2493907977 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 msgr2=0x55eece84c7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece84c7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.896+0000 7f076d926700 1 -- 172.21.15.90:0/2493907977 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0768069120 con 0x55eece84cde0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f076d926700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece84a130 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f07600044c0 tx=0x7f076009a690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a8556f4660a5f7bd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f0750ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f076007d070 con 0x55eece84cde0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f076009adf0 con 0x55eece84cde0 2024-04-06T14:07:42.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55eece754380 con 0x55eece84cde0 2024-04-06T14:07:42.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f076009b050 con 0x55eece84cde0 2024-04-06T14:07:42.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f077b53db80 10 client.25420 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:42.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.897+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55eece5c1b60 con 0x55eece84cde0 2024-04-06T14:07:42.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55eece754380 con 0x55eece84cde0 2024-04-06T14:07:42.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0760080030 con 0x55eece84cde0 2024-04-06T14:07:42.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 10 client.25420 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:42.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 10 client.25420 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:07:42.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 20 client.25420 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8533e0 0x55eece855800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 4 client.25420 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.898+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55eece78c700 con 0x55eece8533e0 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.899+0000 7f076d125700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8533e0 0x55eece855800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.900+0000 7f076d125700 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8533e0 0x55eece855800 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=38d1ec035874c118 server_cookie=9951460c68590dec in_seq=0 out_seq=0 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.900+0000 7f0750ff9700 10 client.25420 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.900+0000 7f0750ff9700 1 -- 172.21.15.90:0/2493907977 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55eece78c700 con 0x55eece8533e0 2024-04-06T14:07:42.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.900+0000 7f0750ff9700 10 client.25420 handle_command_reply: tid=0 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.900+0000 7f077b53db80 1 client.25420 shutdown 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 msgr2=0x55eece84a130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece84a130 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f07600044c0 tx=0x7f076009a690 comp rx=0 tx=0).stop 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 shutdown_connections 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8533e0 0x55eece855800 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece84a130 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece84c7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f0750ff9700 0 client.25420 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 --2- 172.21.15.90:0/2493907977 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece849b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 >> 172.21.15.90:0/2493907977 conn(0x55eece7a16a0 msgr2=0x55eece83cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 shutdown_connections 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 1 -- 172.21.15.90:0/2493907977 wait complete. 2024-04-06T14:07:42.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.901+0000 7f077b53db80 20 client.25420 trim_cache size 0 max 16384 2024-04-06T14:07:42.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 Processor -- start 2024-04-06T14:07:42.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 -- start start 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 0x55eece84f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece85a170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece85a6b0 con 0x55eece8430a0 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece83f210 con 0x55eece842780 2024-04-06T14:07:42.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.917+0000 7f077b53db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece8456d0 con 0x55eece84cde0 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076d125700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 0x55eece84f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076d125700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35040/0 (socket says 172.21.15.90:35040) 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076d125700 1 -- 172.21.15.90:0/431487839 learned_addr learned my addr 172.21.15.90:0/431487839 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076d926700 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece85a170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 -- 172.21.15.90:0/431487839 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 msgr2=0x55eece85a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece85a170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 -- 172.21.15.90:0/431487839 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 msgr2=0x55eece842b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.918+0000 7f076c924700 1 -- 172.21.15.90:0/431487839 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eece759ea0 con 0x55eece8430a0 2024-04-06T14:07:42.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.919+0000 7f076c924700 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 0x55eece84f1d0 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x55eece84b0a0 tx=0x7f0764009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad6701fd55ce0ad6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.919+0000 7f0753fff700 1 -- 172.21.15.90:0/431487839 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0764017070 con 0x55eece8430a0 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.919+0000 7f0753fff700 1 -- 172.21.15.90:0/431487839 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0764018040 con 0x55eece8430a0 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.919+0000 7f0753fff700 1 -- 172.21.15.90:0/431487839 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0764003cc0 con 0x55eece8430a0 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- 172.21.15.90:0/431487839 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 msgr2=0x55eece84f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 0x55eece84f1d0 secure :-1 s=READY pgs=499 cs=0 l=1 rev1=1 crypto rx=0x55eece84b0a0 tx=0x7f0764009260 comp rx=0 tx=0).stop 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- 172.21.15.90:0/431487839 shutdown_connections 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece85a170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece842780 0x55eece842b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 --2- 172.21.15.90:0/431487839 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece8430a0 0x55eece84f1d0 unknown :-1 s=CLOSED pgs=499 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- 172.21.15.90:0/431487839 >> 172.21.15.90:0/431487839 conn(0x55eece7a16a0 msgr2=0x55eece876260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- 172.21.15.90:0/431487839 shutdown_connections 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- 172.21.15.90:0/431487839 wait complete. 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.920+0000 7f077b53db80 1 -- start start 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 Processor -- start 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece8582d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece858810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece8ba850 con 0x55eece842780 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece8ba9c0 con 0x55eece8430a0 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eece8bab80 con 0x55eece84cde0 2024-04-06T14:07:42.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f077b53db80 10 client.0 _ll_register_callbacks cb 0x55eece739470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f076d926700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f076d926700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34958/0 (socket says 172.21.15.90:34958) 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f076d926700 1 -- 172.21.15.90:0/2886255943 learned_addr learned my addr 172.21.15.90:0/2886255943 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.921+0000 7f076c924700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece858810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d125700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece8582d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 -- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 msgr2=0x55eece858810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece858810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 -- 172.21.15.90:0/2886255943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 msgr2=0x55eece8582d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece8582d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 -- 172.21.15.90:0/2886255943 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f07640099f0 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d125700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece8582d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076c924700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece858810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f076d926700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f07600cb570 tx=0x7f07600a3270 comp rx=0 tx=0).ready entity=mon.1 client_cookie=43bc333ce2c1aa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f0750ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07600b6ef0 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f076009adf0 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.922+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f076009ec20 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f076007d050 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 10 client.25426 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55eece619cd0 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f076009ec20 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0760080030 con 0x55eece84cde0 2024-04-06T14:07:42.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 10 client.25426 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 10 client.25426 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 20 client.25426 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8c1f00 0x55eece8c42f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 4 client.25426 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55eece5e2cf0 con 0x55eece8c1f00 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.923+0000 7f076d125700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8c1f00 0x55eece8c42f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f0750ff9700 10 client.25426 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f076d125700 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8c1f00 0x55eece8c42f0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f9882f12d1782696 server_cookie=9a401953637f7667 in_seq=0 out_seq=0 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f0750ff9700 1 -- 172.21.15.90:0/2886255943 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55eece5e2cf0 con 0x55eece8c1f00 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f0750ff9700 10 client.25426 handle_command_reply: tid=0 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 client.25426 shutdown 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 msgr2=0x55eece8ba2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f07600cb570 tx=0x7f07600a3270 comp rx=0 tx=0).stop 2024-04-06T14:07:42.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 shutdown_connections 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eece8c1f00 0x55eece8c42f0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eece84cde0 0x55eece8ba2d0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eece842780 0x55eece8582d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 --2- 172.21.15.90:0/2886255943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eece8430a0 0x55eece858810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 >> 172.21.15.90:0/2886255943 conn(0x55eece7a16a0 msgr2=0x55eece841770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.924+0000 7f0750ff9700 0 client.25426 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.925+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 shutdown_connections 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.925+0000 7f077b53db80 1 -- 172.21.15.90:0/2886255943 wait complete. 2024-04-06T14:07:42.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.926+0000 7f077b53db80 20 client.25426 trim_cache size 0 max 16384 2024-04-06T14:07:42.930 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 msgr2=0x7f07540a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 0x7f07540a3850 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f075c0098e0 tx=0x7f075c003680 comp rx=0 tx=0).stop 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 msgr2=0x7f076819a920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 secure :-1 s=READY pgs=497 cs=0 l=1 rev1=1 crypto rx=0x7f076000e580 tx=0x7f076000e940 comp rx=0 tx=0).stop 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 shutdown_connections 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f07540a1390 0x7f07540a3850 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0768105da0 0x7f076819a920 unknown :-1 s=CLOSED pgs=497 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f07680fff00 0x7f0768196040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 --2- 172.21.15.90:0/4039488209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07681008c0 0x7f0768196580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:42.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 >> 172.21.15.90:0/4039488209 conn(0x7f07680769b0 msgr2=0x7f07680fed70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:42.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 shutdown_connections 2024-04-06T14:07:42.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:42.927+0000 7f076f389700 1 -- 172.21.15.90:0/4039488209 wait complete. 2024-04-06T14:07:42.947 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:42.947 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:07:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 115.0 s, 3395.7 tps, lat 29.456 ms stddev 33.017 2024-04-06T14:07:43.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 -- 172.21.15.90:0/4059131061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 msgr2=0x7f8a2410cad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 --2- 172.21.15.90:0/4059131061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a2410cad0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c00b0a0 tx=0x7f8a1c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 -- 172.21.15.90:0/4059131061 shutdown_connections 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 --2- 172.21.15.90:0/4059131061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a2410cad0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 --2- 172.21.15.90:0/4059131061 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 0x7f8a24106180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 --2- 172.21.15.90:0/4059131061 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 -- 172.21.15.90:0/4059131061 >> 172.21.15.90:0/4059131061 conn(0x7f8a240769b0 msgr2=0x7f8a24076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 -- 172.21.15.90:0/4059131061 shutdown_connections 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.117+0000 7f8a2c026700 1 -- 172.21.15.90:0/4059131061 wait complete. 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 Processor -- start 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 -- start start 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24199550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 0x7f8a2419de30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a2419e470 con 0x7f8a24105da0 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a2419e5e0 con 0x7f8a241008c0 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2c026700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a2419e7a0 con 0x7f8a240fff00 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a295c1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a295c1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35068/0 (socket says 172.21.15.90:35068) 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a295c1700 1 -- 172.21.15.90:0/2445171153 learned_addr learned my addr 172.21.15.90:0/2445171153 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 0x7f8a2419de30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.118+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24199550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 -- 172.21.15.90:0/2445171153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 msgr2=0x7f8a24199550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24199550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 -- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 msgr2=0x7f8a2419de30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 0x7f8a2419de30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 -- 172.21.15.90:0/2445171153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a1c009d00 con 0x7f8a241008c0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24199550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f8a1400e9a0 tx=0x7f8a1400ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a1affd700 1 -- 172.21.15.90:0/2445171153 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a14009790 con 0x7f8a241008c0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a1affd700 1 -- 172.21.15.90:0/2445171153 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a1400cbd0 con 0x7f8a241008c0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a1affd700 1 -- 172.21.15.90:0/2445171153 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a14018680 con 0x7f8a241008c0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a2419ea80 con 0x7f8a241008c0 2024-04-06T14:07:43.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.119+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a241a62c0 con 0x7f8a241008c0 2024-04-06T14:07:43.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.124+0000 7f8a381dab80 1 Processor -- start 2024-04-06T14:07:43.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.124+0000 7f8a381dab80 1 -- start start 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b610a0 0x555c10b6a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b6d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10ab8990 con 0x555c10b610a0 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10ab6350 con 0x555c10b6ade0 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a381dab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10ab1c40 con 0x555c10b60780 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a1affd700 1 -- 172.21.15.90:0/2445171153 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8a1400cd40 con 0x7f8a241008c0 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a2a5c3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a2a5c3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34998/0 (socket says 172.21.15.90:34998) 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a2a5c3700 1 -- 172.21.15.90:0/3094575005 learned_addr learned my addr 172.21.15.90:0/3094575005 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b610a0 0x555c10b6a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a295c1700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b6d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.125+0000 7f8a2a5c3700 1 -- 172.21.15.90:0/3094575005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 msgr2=0x555c10b6d1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b6d1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a2a5c3700 1 -- 172.21.15.90:0/3094575005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b610a0 msgr2=0x555c10b6a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b610a0 0x555c10b6a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a2a5c3700 1 -- 172.21.15.90:0/3094575005 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555c109333b0 con 0x555c10b60780 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a295c1700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b6d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c015000 tx=0x7f8a1c009430 comp rx=0 tx=0).ready entity=mon.1 client_cookie=803c87210aa7419f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a1affd700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 0x7f8a100a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a0ffff700 1 -- 172.21.15.90:0/3094575005 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c00e040 con 0x555c10b60780 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a0ffff700 1 -- 172.21.15.90:0/3094575005 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a1c007e30 con 0x555c10b60780 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a0ffff700 1 -- 172.21.15.90:0/3094575005 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a1c0079a0 con 0x555c10b60780 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 0x7f8a100a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.126+0000 7f8a1affd700 1 -- 172.21.15.90:0/2445171153 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a14014070 con 0x7f8a241008c0 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 -- 172.21.15.90:0/3094575005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 msgr2=0x555c10b60b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 0x7f8a100a37d0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f8a20006fd0 tx=0x7f8a20008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f8a1c015000 tx=0x7f8a1c009430 comp rx=0 tx=0).stop 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 -- 172.21.15.90:0/3094575005 shutdown_connections 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b6d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b60780 0x555c10b60b60 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3094575005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b610a0 0x555c10b6a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 -- 172.21.15.90:0/3094575005 >> 172.21.15.90:0/3094575005 conn(0x555c10abf6a0 msgr2=0x555c10b5ba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 -- 172.21.15.90:0/3094575005 shutdown_connections 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.127+0000 7f8a381dab80 1 -- 172.21.15.90:0/3094575005 wait complete. 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 -- start start 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 Processor -- start 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10b67bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b680f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10b68630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10a4dd80 con 0x555c10b6ade0 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10a4da20 con 0x555c10b60780 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c109356a0 con 0x555c10b610a0 2024-04-06T14:07:43.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.128+0000 7f8a381dab80 10 client.0 _ll_register_callbacks cb 0x555c10a57470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a2a5c3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10b68630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a2a5c3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10b68630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34408/0 (socket says 172.21.15.90:34408) 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a2a5c3700 1 -- 172.21.15.90:0/2525633247 learned_addr learned my addr 172.21.15.90:0/2525633247 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b680f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 -- 172.21.15.90:0/2525633247 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 msgr2=0x555c10b67bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10b67bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 -- 172.21.15.90:0/2525633247 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 msgr2=0x555c10b68630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10b68630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.129+0000 7f8a295c1700 1 -- 172.21.15.90:0/2525633247 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a24069320 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a295c1700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b680f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f8a140ca630 tx=0x7f8a140cab90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e7f56897a179825c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a0d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a1409de20 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a140a2470 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555c10a72380 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a140a1fe0 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a381dab80 10 client.25438 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555c108dfb60 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.131+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555c10a72380 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a140afd90 con 0x555c10b610a0 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 10 client.25438 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:43.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 10 client.25438 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 20 client.25438 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b940e0 0x555c10b96500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 4 client.25438 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555c10aaa700 con 0x555c10b940e0 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b940e0 0x555c10b96500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.132+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b940e0 0x555c10b96500 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ff98891570417907 server_cookie=80634c4becc0ff8f in_seq=0 out_seq=0 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.133+0000 7f8a0d7fa700 10 client.25438 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.133+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/2525633247 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555c10aaa700 con 0x555c10b940e0 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.133+0000 7f8a0d7fa700 10 client.25438 handle_command_reply: tid=0 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.133+0000 7f8a381dab80 1 client.25438 shutdown 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 msgr2=0x555c10b680f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b680f0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f8a140ca630 tx=0x7f8a140cab90 comp rx=0 tx=0).stop 2024-04-06T14:07:43.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 shutdown_connections 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b940e0 0x555c10b96500 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10b68630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10b67bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 --2- 172.21.15.90:0/2525633247 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b680f0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 >> 172.21.15.90:0/2525633247 conn(0x555c10abf6a0 msgr2=0x555c10b5ac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a0d7fa700 0 client.25438 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 shutdown_connections 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 1 -- 172.21.15.90:0/2525633247 wait complete. 2024-04-06T14:07:43.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.134+0000 7f8a381dab80 20 client.25438 trim_cache size 0 max 16384 2024-04-06T14:07:43.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.152+0000 7f8a381dab80 1 Processor -- start 2024-04-06T14:07:43.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 -- start start 2024-04-06T14:07:43.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b6a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b7a290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10b6a700 con 0x555c10b60780 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10b5c650 con 0x555c10b6ade0 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a381dab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10b7a7d0 con 0x555c10b610a0 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34424/0 (socket says 172.21.15.90:34424) 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 -- 172.21.15.90:0/1744381657 learned_addr learned my addr 172.21.15.90:0/1744381657 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b7a290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a295c1700 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b6a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 -- 172.21.15.90:0/1744381657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 msgr2=0x555c10b6a1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b6a1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 -- 172.21.15.90:0/1744381657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 msgr2=0x555c10b7a290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b7a290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.153+0000 7f8a29dc2700 1 -- 172.21.15.90:0/1744381657 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555c10a77ea0 con 0x555c10b60780 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.154+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f8a2000bfd0 tx=0x7f8a20004860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3685291a66872c62 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.154+0000 7f8a18ff9700 1 -- 172.21.15.90:0/1744381657 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a2000fda0 con 0x555c10b60780 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.154+0000 7f8a18ff9700 1 -- 172.21.15.90:0/1744381657 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a20024500 con 0x555c10b60780 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.154+0000 7f8a18ff9700 1 -- 172.21.15.90:0/1744381657 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a2002de80 con 0x555c10b60780 2024-04-06T14:07:43.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 -- 172.21.15.90:0/1744381657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 msgr2=0x555c10b902d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 secure :-1 s=READY pgs=500 cs=0 l=1 rev1=1 crypto rx=0x7f8a2000bfd0 tx=0x7f8a20004860 comp rx=0 tx=0).stop 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 -- 172.21.15.90:0/1744381657 shutdown_connections 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b6ade0 0x555c10b7a290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b60780 0x555c10b902d0 unknown :-1 s=CLOSED pgs=500 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 --2- 172.21.15.90:0/1744381657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10b6a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.155+0000 7f8a381dab80 1 -- 172.21.15.90:0/1744381657 >> 172.21.15.90:0/1744381657 conn(0x555c10abf6a0 msgr2=0x555c10b62e80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.156+0000 7f8a381dab80 1 -- 172.21.15.90:0/1744381657 shutdown_connections 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.156+0000 7f8a381dab80 1 -- 172.21.15.90:0/1744381657 wait complete. 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 -- start start 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 Processor -- start 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10be3420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10bdfa30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10be00d0 con 0x555c10b6ade0 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10be0240 con 0x555c10b60780 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c10be0400 con 0x555c10b610a0 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.157+0000 7f8a381dab80 10 client.0 _ll_register_callbacks cb 0x555c10a57470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:35020/0 (socket says 172.21.15.90:35020) 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 -- 172.21.15.90:0/3033401657 learned_addr learned my addr 172.21.15.90:0/3033401657 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10be3420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a2a5c3700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10bdfa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 -- 172.21.15.90:0/3033401657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 msgr2=0x555c10be3420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10be3420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 -- 172.21.15.90:0/3033401657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 msgr2=0x555c10bdfa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10bdfa30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 -- 172.21.15.90:0/3033401657 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a2000e040 con 0x555c10b610a0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a295c1700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f8a14099120 tx=0x7f8a14098f50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d34639f301f78272 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a0d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a140b7070 con 0x555c10b610a0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a140a2470 con 0x555c10b610a0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.158+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8a1c007e30 con 0x555c10b610a0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a1409de20 con 0x555c10b610a0 2024-04-06T14:07:43.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 10 client.25444 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555c10937cd0 con 0x555c10b610a0 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a14083080 con 0x555c10b610a0 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a1407d030 con 0x555c10b610a0 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 10 client.25444 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 10 client.25444 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 20 client.25444 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b76910 0x555c10b78d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 4 client.25444 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x555c10900cf0 con 0x555c10b76910 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.159+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b76910 0x555c10b78d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a0d7fa700 10 client.25444 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a29dc2700 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b76910 0x555c10b78d30 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c19e94e19e09c4c7 server_cookie=d8ec77823585c239 in_seq=0 out_seq=0 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a0d7fa700 1 -- 172.21.15.90:0/3033401657 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x555c10900cf0 con 0x555c10b76910 2024-04-06T14:07:43.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a0d7fa700 10 client.25444 handle_command_reply: tid=0 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 client.25444 shutdown 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 msgr2=0x555c10be3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f8a14099120 tx=0x7f8a14098f50 comp rx=0 tx=0).stop 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 shutdown_connections 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x555c10b76910 0x555c10b78d30 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555c10b6ade0 0x555c10bdfa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555c10b60780 0x555c10be3420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 --2- 172.21.15.90:0/3033401657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555c10b610a0 0x555c10be3960 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 >> 172.21.15.90:0/3033401657 conn(0x555c10abf6a0 msgr2=0x555c10b66310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a0d7fa700 0 client.25444 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 shutdown_connections 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.160+0000 7f8a381dab80 1 -- 172.21.15.90:0/3033401657 wait complete. 2024-04-06T14:07:43.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a381dab80 20 client.25444 trim_cache size 0 max 16384 2024-04-06T14:07:43.165 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:43.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 msgr2=0x7f8a100a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 0x7f8a100a37d0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f8a20006fd0 tx=0x7f8a20008040 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 msgr2=0x7f8a24199a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f8a1400e9a0 tx=0x7f8a1400ecb0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 shutdown_connections 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a100a1310 0x7f8a100a37d0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a24105da0 0x7f8a2419de30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a240fff00 0x7f8a24199550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 --2- 172.21.15.90:0/2445171153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a241008c0 0x7f8a24199a90 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 >> 172.21.15.90:0/2445171153 conn(0x7f8a240769b0 msgr2=0x7f8a240fec00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 shutdown_connections 2024-04-06T14:07:43.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.161+0000 7f8a2c026700 1 -- 172.21.15.90:0/2445171153 wait complete. 2024-04-06T14:07:43.180 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:07:43.180 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:07:43.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 -- 172.21.15.90:0/2761217053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 msgr2=0x7f31b010dc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/2761217053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b010dc20 secure :-1 s=READY pgs=501 cs=0 l=1 rev1=1 crypto rx=0x7f31a800b0d0 tx=0x7f31a800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 -- 172.21.15.90:0/2761217053 shutdown_connections 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/2761217053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b010dc20 unknown :-1 s=CLOSED pgs=501 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/2761217053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f31b00680f0 0x7f31b00684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/2761217053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b0068aa0 0x7f31b0105970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 -- 172.21.15.90:0/2761217053 >> 172.21.15.90:0/2761217053 conn(0x7f31b0076d50 msgr2=0x7f31b0077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 -- 172.21.15.90:0/2761217053 shutdown_connections 2024-04-06T14:07:43.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.380+0000 7f31b4b9d700 1 -- 172.21.15.90:0/2761217053 wait complete. 2024-04-06T14:07:43.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.381+0000 7f31b4b9d700 1 Processor -- start 2024-04-06T14:07:43.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.381+0000 7f31b4b9d700 1 -- start start 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b0197730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f31b019bad0 0x7f31b019bf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31b019c550 con 0x7f31b01060a0 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31b019c6c0 con 0x7f31b00680f0 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31b4b9d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31b019c880 con 0x7f31b019bad0 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59964/0 (socket says 172.21.15.90:59964) 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 -- 172.21.15.90:0/1488738507 learned_addr learned my addr 172.21.15.90:0/1488738507 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31add9b700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b0197730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 -- 172.21.15.90:0/1488738507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f31b019bad0 msgr2=0x7f31b019bf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f31b019bad0 0x7f31b019bf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 -- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 msgr2=0x7f31b0197730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b0197730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.382+0000 7f31ae59c700 1 -- 172.21.15.90:0/1488738507 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31a8009d00 con 0x7f31b00680f0 2024-04-06T14:07:43.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f31ae59c700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f31a000d950 tx=0x7f31a000dd10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f319f7fe700 1 -- 172.21.15.90:0/1488738507 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a0009b40 con 0x7f31b00680f0 2024-04-06T14:07:43.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f319f7fe700 1 -- 172.21.15.90:0/1488738507 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f31a000b6e0 con 0x7f31b00680f0 2024-04-06T14:07:43.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f31b01a3de0 con 0x7f31b00680f0 2024-04-06T14:07:43.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f319f7fe700 1 -- 172.21.15.90:0/1488738507 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a000f620 con 0x7f31b00680f0 2024-04-06T14:07:43.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.383+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31b01a4260 con 0x7f31b00680f0 2024-04-06T14:07:43.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.389+0000 7f31c0d51b80 1 Processor -- start 2024-04-06T14:07:43.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.389+0000 7f319f7fe700 1 -- 172.21.15.90:0/1488738507 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f31a0010460 con 0x7f31b00680f0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.389+0000 7f319f7fe700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 0x7f31980a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.390+0000 7f319f7fe700 1 -- 172.21.15.90:0/1488738507 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f31a00ccfc0 con 0x7f31b00680f0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.390+0000 7f31aed9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 0x7f31980a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.390+0000 7f31c0d51b80 1 -- start start 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.390+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.390+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e228a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 0x55b7c7e361e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7d70990 con 0x55b7c7e33df0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7d6e350 con 0x55b7c7e18780 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7d69c40 con 0x55b7c7e190a0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31add9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e228a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59974/0 (socket says 172.21.15.90:59974) 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 -- 172.21.15.90:0/3172283486 learned_addr learned my addr 172.21.15.90:0/3172283486 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 -- 172.21.15.90:0/3172283486 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 msgr2=0x55b7c7e228a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e228a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31aed9d700 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 0x55b7c7e361e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 -- 172.21.15.90:0/3172283486 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 msgr2=0x55b7c7e361e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 0x55b7c7e361e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 -- 172.21.15.90:0/3172283486 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b7c7beb3b0 con 0x55b7c7e18780 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31aed9d700 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 0x55b7c7e361e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31aed9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 0x7f31980a3960 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f31a8009fd0 tx=0x7f31a8009570 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f31ae59c700 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f31a00a1ff0 tx=0x7f31a00963e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=24cc60e638aa5d87 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f319cff9700 1 -- 172.21.15.90:0/3172283486 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a0099a00 con 0x55b7c7e18780 2024-04-06T14:07:43.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f319cff9700 1 -- 172.21.15.90:0/3172283486 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f31a009a7a0 con 0x55b7c7e18780 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.391+0000 7f319cff9700 1 -- 172.21.15.90:0/3172283486 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a0096980 con 0x55b7c7e18780 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.392+0000 7f31c0d51b80 1 -- 172.21.15.90:0/3172283486 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 msgr2=0x55b7c7e18b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.392+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f31a00a1ff0 tx=0x7f31a00963e0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.394+0000 7f31c0d51b80 1 -- 172.21.15.90:0/3172283486 shutdown_connections 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.394+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e33df0 0x55b7c7e361e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.394+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e18b60 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.394+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/3172283486 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e228a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.394+0000 7f31c0d51b80 1 -- 172.21.15.90:0/3172283486 >> 172.21.15.90:0/3172283486 conn(0x55b7c7d776a0 msgr2=0x55b7c7e13a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.396+0000 7f31c0d51b80 1 -- 172.21.15.90:0/3172283486 shutdown_connections 2024-04-06T14:07:43.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.396+0000 7f31c0d51b80 1 -- 172.21.15.90:0/3172283486 wait complete. 2024-04-06T14:07:43.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.396+0000 7f31c0d51b80 1 -- start start 2024-04-06T14:07:43.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.396+0000 7f31c0d51b80 1 Processor -- start 2024-04-06T14:07:43.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.397+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e1fbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.397+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e20100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.397+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.397+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7d05d80 con 0x55b7c7e18780 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.397+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7d05a20 con 0x55b7c7e33df0 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7bed6a0 con 0x55b7c7e190a0 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31c0d51b80 10 client.0 _ll_register_callbacks cb 0x55b7c7d0f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59990/0 (socket says 172.21.15.90:59990) 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 -- 172.21.15.90:0/4245489191 learned_addr learned my addr 172.21.15.90:0/4245489191 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31add9b700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e20100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 -- 172.21.15.90:0/4245489191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 msgr2=0x55b7c7e20100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e20100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 -- 172.21.15.90:0/4245489191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 msgr2=0x55b7c7e1fbc0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e1fbc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 -- 172.21.15.90:0/4245489191 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31b01a4260 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31aed9d700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f31a8008ff0 tx=0x7f31a800b730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4d9bfa0940442c6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f3195ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a800e040 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f31a80079e0 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.398+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b7c7d2a380 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f31a8029890 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 10 client.25514 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b7c7b97b60 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b7c7d2a380 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f31a801db60 con 0x55b7c7e33df0 2024-04-06T14:07:43.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 10 client.25514 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 10 client.25514 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 20 client.25514 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7e3d9b0 0x55b7c7e3fdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 4 client.25514 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b7c7d62700 con 0x55b7c7e3d9b0 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.399+0000 7f31ae59c700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7e3d9b0 0x55b7c7e3fdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.400+0000 7f3195ffb700 10 client.25514 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.400+0000 7f31ae59c700 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7e3d9b0 0x55b7c7e3fdd0 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c3486e59836b0b2c server_cookie=8218c3954740190f in_seq=0 out_seq=0 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f3195ffb700 1 -- 172.21.15.90:0/4245489191 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b7c7d62700 con 0x55b7c7e3d9b0 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f3195ffb700 10 client.25514 handle_command_reply: tid=0 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 client.25514 shutdown 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 msgr2=0x55b7c7e20640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f31a8008ff0 tx=0x7f31a800b730 comp rx=0 tx=0).stop 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 shutdown_connections 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20640 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e1fbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7e3d9b0 0x55b7c7e3fdd0 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/4245489191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e20100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f3195ffb700 0 client.25514 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.401+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 >> 172.21.15.90:0/4245489191 conn(0x55b7c7d776a0 msgr2=0x55b7c7e12c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.402+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 shutdown_connections 2024-04-06T14:07:43.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.402+0000 7f31c0d51b80 1 -- 172.21.15.90:0/4245489191 wait complete. 2024-04-06T14:07:43.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.402+0000 7f31c0d51b80 20 client.25514 trim_cache size 0 max 16384 2024-04-06T14:07:43.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.422+0000 7f31c0d51b80 1 Processor -- start 2024-04-06T14:07:43.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 -- start start 2024-04-06T14:07:43.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e39200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 0x55b7c7e13c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7e151f0 con 0x55b7c7e190a0 2024-04-06T14:07:43.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7e14fd0 con 0x55b7c7e18780 2024-04-06T14:07:43.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.423+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7e14dc0 con 0x55b7c7e33df0 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31aed9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31aed9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:46234/0 (socket says 172.21.15.90:46234) 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31aed9d700 1 -- 172.21.15.90:0/1518284099 learned_addr learned my addr 172.21.15.90:0/1518284099 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 0x55b7c7e13c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 -- 172.21.15.90:0/1518284099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 msgr2=0x55b7c7e20110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 -- 172.21.15.90:0/1518284099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 msgr2=0x55b7c7e39200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e39200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.424+0000 7f31add9b700 1 -- 172.21.15.90:0/1518284099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b7c7d2fea0 con 0x55b7c7e190a0 2024-04-06T14:07:43.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.425+0000 7f31add9b700 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 0x55b7c7e13c30 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f31b0198740 tx=0x7f31a4009990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7aa09ce422b80d95 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.425+0000 7f319d7fa700 1 -- 172.21.15.90:0/1518284099 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a4003880 con 0x55b7c7e190a0 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.425+0000 7f319d7fa700 1 -- 172.21.15.90:0/1518284099 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f31a4017040 con 0x55b7c7e190a0 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.425+0000 7f319d7fa700 1 -- 172.21.15.90:0/1518284099 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a4011d70 con 0x55b7c7e190a0 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1518284099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 msgr2=0x55b7c7e13c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 0x55b7c7e13c30 secure :-1 s=READY pgs=502 cs=0 l=1 rev1=1 crypto rx=0x7f31b0198740 tx=0x7f31a4009990 comp rx=0 tx=0).stop 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1518284099 shutdown_connections 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e33df0 0x55b7c7e20110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e18780 0x55b7c7e39200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1518284099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e190a0 0x55b7c7e13c30 unknown :-1 s=CLOSED pgs=502 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1518284099 >> 172.21.15.90:0/1518284099 conn(0x55b7c7d776a0 msgr2=0x55b7c7e13520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1518284099 shutdown_connections 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1518284099 wait complete. 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 -- start start 2024-04-06T14:07:43.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.426+0000 7f31c0d51b80 1 Processor -- start 2024-04-06T14:07:43.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e39060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e36610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e36b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7e37230 con 0x55b7c7e18780 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7e373a0 con 0x55b7c7e33df0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7c7ea86f0 con 0x55b7c7e190a0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31c0d51b80 10 client.0 _ll_register_callbacks cb 0x55b7c7d0f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e39060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e36610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31ae59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e39060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45084/0 (socket says 172.21.15.90:45084) 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31ae59c700 1 -- 172.21.15.90:0/1163237693 learned_addr learned my addr 172.21.15.90:0/1163237693 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31aed9d700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e36b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 -- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 msgr2=0x55b7c7e36b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e36b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 -- 172.21.15.90:0/1163237693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 msgr2=0x55b7c7e39060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:43.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e39060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31add9b700 1 -- 172.21.15.90:0/1163237693 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31a4004320 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.427+0000 7f31aed9d700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e36b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31add9b700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e36610 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f31a40118f0 tx=0x7f31a4011670 comp rx=0 tx=0).ready entity=mon.1 client_cookie=58c222867fca3005 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f31a4003a90 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f31a4003bf0 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f31a009a7a0 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f31a4003db0 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 10 client.25459 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b7c7befcd0 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f31a009a7a0 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f31a401b510 con 0x55b7c7e190a0 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 10 client.25459 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 10 client.25459 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:43.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 20 client.25459 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:43.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7ea9e10 0x55b7c7eac230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:43.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 4 client.25459 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:07:43.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.428+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b7c7bb8cf0 con 0x55b7c7ea9e10 2024-04-06T14:07:43.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.430+0000 7f31ae59c700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7ea9e10 0x55b7c7eac230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:43.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.431+0000 7f3195ffb700 10 client.25459 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.431+0000 7f31ae59c700 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7ea9e10 0x55b7c7eac230 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=558c0023de8bd4c1 server_cookie=ceb904066e8de1ea in_seq=0 out_seq=0 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.432+0000 7f3195ffb700 1 -- 172.21.15.90:0/1163237693 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b7c7bb8cf0 con 0x55b7c7ea9e10 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.432+0000 7f3195ffb700 10 client.25459 handle_command_reply: tid=0 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.432+0000 7f31c0d51b80 1 client.25459 shutdown 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.432+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 msgr2=0x55b7c7e36610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.432+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e36610 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f31a40118f0 tx=0x7f31a4011670 comp rx=0 tx=0).stop 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 shutdown_connections 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b7c7e33df0 0x55b7c7e36b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b7c7e18780 0x55b7c7e39060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b7c7ea9e10 0x55b7c7eac230 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 --2- 172.21.15.90:0/1163237693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b7c7e190a0 0x55b7c7e36610 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f3195ffb700 0 client.25459 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:43.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 >> 172.21.15.90:0/1163237693 conn(0x55b7c7d776a0 msgr2=0x55b7c7e48710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 shutdown_connections 2024-04-06T14:07:43.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 1 -- 172.21.15.90:0/1163237693 wait complete. 2024-04-06T14:07:43.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.433+0000 7f31c0d51b80 20 client.25459 trim_cache size 0 max 16384 2024-04-06T14:07:43.435 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 msgr2=0x7f31980a3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 0x7f31980a3960 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f31a8009fd0 tx=0x7f31a8009570 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 msgr2=0x7f31b01971f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f31a000d950 tx=0x7f31a000dd10 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 shutdown_connections 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f31980a14a0 0x7f31980a3960 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f31b019bad0 0x7f31b019bf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f31b00680f0 0x7f31b01971f0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 --2- 172.21.15.90:0/1488738507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f31b01060a0 0x7f31b0197730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.434+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 >> 172.21.15.90:0/1488738507 conn(0x7f31b0076d50 msgr2=0x7f31b00fdf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:43.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.435+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 shutdown_connections 2024-04-06T14:07:43.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:43.435+0000 7f31b4b9d700 1 -- 172.21.15.90:0/1488738507 wait complete. 2024-04-06T14:07:43.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:43 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1547025046' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:43.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:43 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1547025046' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:44.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:43 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1547025046' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:07:44.452 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:07:44.452 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:07:44.452 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:07:44.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.646+0000 7f28f43d1700 1 -- 172.21.15.90:0/3117143177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec1088b0 msgr2=0x7f28ec108c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.646+0000 7f28f43d1700 1 --2- 172.21.15.90:0/3117143177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec1088b0 0x7f28ec108c90 secure :-1 s=READY pgs=503 cs=0 l=1 rev1=1 crypto rx=0x7f28e400aad0 tx=0x7f28e400ade0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 -- 172.21.15.90:0/3117143177 shutdown_connections 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 --2- 172.21.15.90:0/3117143177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec10cf30 0x7f28ec114ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 --2- 172.21.15.90:0/3117143177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec1088b0 0x7f28ec108c90 unknown :-1 s=CLOSED pgs=503 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 --2- 172.21.15.90:0/3117143177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec109260 0x7f28ec10c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 -- 172.21.15.90:0/3117143177 >> 172.21.15.90:0/3117143177 conn(0x7f28ec06be00 msgr2=0x7f28ec06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 -- 172.21.15.90:0/3117143177 shutdown_connections 2024-04-06T14:07:44.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.647+0000 7f28f43d1700 1 -- 172.21.15.90:0/3117143177 wait complete. 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 Processor -- start 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 -- start start 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 0x7f28ec111750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 0x7f28ec112140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28ec114e70 con 0x7f28ec10cf30 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28ec1b28a0 con 0x7f28ec1088b0 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.648+0000 7f28f43d1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28ec1b2a60 con 0x7f28ec111c90 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60018/0 (socket says 172.21.15.90:60018) 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 -- 172.21.15.90:0/642337624 learned_addr learned my addr 172.21.15.90:0/642337624 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f296e700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 0x7f28ec112140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f196c700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 0x7f28ec111750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 -- 172.21.15.90:0/642337624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 msgr2=0x7f28ec112140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 0x7f28ec112140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 -- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 msgr2=0x7f28ec111750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 0x7f28ec111750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 -- 172.21.15.90:0/642337624 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28e400a700 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.649+0000 7f28f216d700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f28e4006010 tx=0x7f28e40040b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28eb7fe700 1 -- 172.21.15.90:0/642337624 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e4004380 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28ec1b2ce0 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28ec1b3160 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28eb7fe700 1 -- 172.21.15.90:0/642337624 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28e4003890 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28eb7fe700 1 -- 172.21.15.90:0/642337624 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e4016680 con 0x7f28ec1088b0 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28f296e700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 0x7f28ec112140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:44.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.650+0000 7f28f196c700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 0x7f28ec111750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:44.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.655+0000 7f28eb7fe700 1 -- 172.21.15.90:0/642337624 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f28e400e450 con 0x7f28ec1088b0 2024-04-06T14:07:44.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.656+0000 7f28eb7fe700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 0x7f28d40a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.656+0000 7f28eb7fe700 1 -- 172.21.15.90:0/642337624 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28e4012070 con 0x7f28ec1088b0 2024-04-06T14:07:44.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.656+0000 7f28f196c700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 0x7f28d40a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.657+0000 7f2900585b80 1 Processor -- start 2024-04-06T14:07:44.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.657+0000 7f2900585b80 1 -- start start 2024-04-06T14:07:44.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f196c700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 0x7f28d40a37d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f28e0006fd0 tx=0x7f28e0008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed4920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed4e60 0x55e38fed7250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fe22a10 con 0x55e38fed4e60 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fe203d0 con 0x55e38feca800 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f2900585b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fe1bcc0 con 0x55e38fecb120 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60024/0 (socket says 172.21.15.90:60024) 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 -- 172.21.15.90:0/992429737 learned_addr learned my addr 172.21.15.90:0/992429737 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f196c700 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed4920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f296e700 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed4e60 0x55e38fed7250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 -- 172.21.15.90:0/992429737 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 msgr2=0x55e38fed4920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed4920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 -- 172.21.15.90:0/992429737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed4e60 msgr2=0x55e38fed7250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed4e60 0x55e38fed7250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 -- 172.21.15.90:0/992429737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e38fc9d430 con 0x55e38feca800 2024-04-06T14:07:44.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28f216d700 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f28e4097de0 tx=0x7f28e409acd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fa5aca1653abc0f1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28e8da5700 1 -- 172.21.15.90:0/992429737 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e409b370 con 0x55e38feca800 2024-04-06T14:07:44.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.658+0000 7f28e8da5700 1 -- 172.21.15.90:0/992429737 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28e409f1a0 con 0x55e38feca800 2024-04-06T14:07:44.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.659+0000 7f28e8da5700 1 -- 172.21.15.90:0/992429737 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e409b810 con 0x55e38feca800 2024-04-06T14:07:44.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.659+0000 7f2900585b80 1 -- 172.21.15.90:0/992429737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 msgr2=0x55e38fecabe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.659+0000 7f2900585b80 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f28e4097de0 tx=0x7f28e409acd0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 -- 172.21.15.90:0/992429737 shutdown_connections 2024-04-06T14:07:44.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed4e60 0x55e38fed7250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38feca800 0x55e38fecabe0 secure :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f28e4097de0 tx=0x7f28e409acd0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 --2- 172.21.15.90:0/992429737 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed4920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 -- 172.21.15.90:0/992429737 >> 172.21.15.90:0/992429737 conn(0x55e38fe29720 msgr2=0x55e38fec5b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 -- 172.21.15.90:0/992429737 shutdown_connections 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.660+0000 7f2900585b80 1 -- 172.21.15.90:0/992429737 wait complete. 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 -- start start 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 Processor -- start 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed1df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fee0930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fdb7e00 con 0x55e38fed2870 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fdb7aa0 con 0x55e38fed4e60 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fc9f720 con 0x55e38fecb120 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f2900585b80 10 client.0 _ll_register_callbacks cb 0x55e38fdc14f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:44.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f28f196c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f28f216d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed1df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f28f196c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60042/0 (socket says 172.21.15.90:60042) 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.661+0000 7f28f196c700 1 -- 172.21.15.90:0/3587600035 learned_addr learned my addr 172.21.15.90:0/3587600035 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 -- 172.21.15.90:0/3587600035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 msgr2=0x55e38fed1df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed1df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 -- 172.21.15.90:0/3587600035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 msgr2=0x55e38fee0930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fee0930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 -- 172.21.15.90:0/3587600035 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28ec1b3160 con 0x55e38fed4e60 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28f196c700 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f28e000cc00 tx=0x7f28e00047e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=82e88c0ad6e5330 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e0013ce0 con 0x55e38fed4e60 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28e0013e40 con 0x55e38fed4e60 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e38fddc400 con 0x55e38fed4e60 2024-04-06T14:07:44.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f28e0028440 con 0x55e38fed4e60 2024-04-06T14:07:44.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.662+0000 7f2900585b80 10 client.25526 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:44.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e38fc49be0 con 0x55e38fed4e60 2024-04-06T14:07:44.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e38fddc400 con 0x55e38fed4e60 2024-04-06T14:07:44.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f28e001e610 con 0x55e38fed4e60 2024-04-06T14:07:44.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 10 client.25526 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:44.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 10 client.25526 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:44.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 20 client.25526 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38fee4fb0 0x55e38fee73d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 4 client.25526 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.663+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e38fe14780 con 0x55e38fee4fb0 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.664+0000 7f28f216d700 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38fee4fb0 0x55e38fee73d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.664+0000 7f28d1ffb700 10 client.25526 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.664+0000 7f28f216d700 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38fee4fb0 0x55e38fee73d0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ea83023ad4065718 server_cookie=fe2e5ba832518f12 in_seq=0 out_seq=0 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.665+0000 7f28d1ffb700 1 -- 172.21.15.90:0/3587600035 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x7f28e40b3e90 con 0x55e38fee4fb0 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.665+0000 7f28d1ffb700 10 client.25526 handle_command_reply: tid=0 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.665+0000 7f2900585b80 1 client.25526 shutdown 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.665+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 msgr2=0x55e38fed2330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.665+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f28e000cc00 tx=0x7f28e00047e0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 shutdown_connections 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fee0930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38fee4fb0 0x55e38fee73d0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38fed1df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 --2- 172.21.15.90:0/3587600035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38fed2330 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f28d1ffb700 0 client.25526 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 >> 172.21.15.90:0/3587600035 conn(0x55e38fe29720 msgr2=0x55e38fec4ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 shutdown_connections 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 1 -- 172.21.15.90:0/3587600035 wait complete. 2024-04-06T14:07:44.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.666+0000 7f2900585b80 20 client.25526 trim_cache size 0 max 16384 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 Processor -- start 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 -- start start 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 0x55e38fed0d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 0x55e38fee0860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 0x55e38fedb880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38febfde0 con 0x55e38fecb120 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fec72d0 con 0x55e38fed2870 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f2900585b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38fed3190 con 0x55e38fed4e60 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f216d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 0x55e38fed0d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f196c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 0x55e38fee0860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f196c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 0x55e38fee0860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60056/0 (socket says 172.21.15.90:60056) 2024-04-06T14:07:44.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f196c700 1 -- 172.21.15.90:0/2713135846 learned_addr learned my addr 172.21.15.90:0/2713135846 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f296e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 0x55e38fedb880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.687+0000 7f28f216d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 0x55e38fed0d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45132/0 (socket says 172.21.15.90:45132) 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 -- 172.21.15.90:0/2713135846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 msgr2=0x55e38fee0860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 0x55e38fee0860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 -- 172.21.15.90:0/2713135846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 msgr2=0x55e38fed0d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 0x55e38fed0d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 -- 172.21.15.90:0/2713135846 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e38fde1f20 con 0x55e38fed4e60 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.688+0000 7f28f296e700 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 0x55e38fedb880 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f28dc008d30 tx=0x7f28dc008d60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c0e0d0b14bca3c7f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f28e95a6700 1 -- 172.21.15.90:0/2713135846 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28dc00da20 con 0x55e38fed4e60 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f28e95a6700 1 -- 172.21.15.90:0/2713135846 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28dc00db80 con 0x55e38fed4e60 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f28e95a6700 1 -- 172.21.15.90:0/2713135846 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28dc00f670 con 0x55e38fed4e60 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 -- 172.21.15.90:0/2713135846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 msgr2=0x55e38fedb880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 0x55e38fedb880 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f28dc008d30 tx=0x7f28dc008d60 comp rx=0 tx=0).stop 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 -- 172.21.15.90:0/2713135846 shutdown_connections 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fed4e60 0x55e38fedb880 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fecb120 0x55e38fed0d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 --2- 172.21.15.90:0/2713135846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed2870 0x55e38fee0860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 -- 172.21.15.90:0/2713135846 >> 172.21.15.90:0/2713135846 conn(0x55e38fe29720 msgr2=0x55e38fed3690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 -- 172.21.15.90:0/2713135846 shutdown_connections 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.689+0000 7f2900585b80 1 -- 172.21.15.90:0/2713135846 wait complete. 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 -- start start 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 Processor -- start 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38feda750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fedac90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38ff38df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38ff39330 con 0x55e38fed2870 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38ff39470 con 0x55e38fed4e60 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e38ff395e0 con 0x55e38fecb120 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f2900585b80 10 client.0 _ll_register_callbacks cb 0x55e38fdc14f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f28f196c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fedac90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f28f296e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38ff38df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f28f196c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fedac90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45136/0 (socket says 172.21.15.90:45136) 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.690+0000 7f28f196c700 1 -- 172.21.15.90:0/2213801118 learned_addr learned my addr 172.21.15.90:0/2213801118 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 -- 172.21.15.90:0/2213801118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 msgr2=0x55e38feda750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38feda750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 -- 172.21.15.90:0/2213801118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 msgr2=0x55e38fedac90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fedac90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 -- 172.21.15.90:0/2213801118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28dc008990 con 0x55e38fed4e60 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28f296e700 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38ff38df0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f28dc018d20 tx=0x7f28dc019fc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6023a81009506f7c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28dc01c070 con 0x55e38fed4e60 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f28dc013070 con 0x55e38fed4e60 2024-04-06T14:07:44.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.691+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f28e409f1a0 con 0x55e38fed4e60 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f28dc018480 con 0x55e38fed4e60 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 10 client.25532 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e38fca1d50 con 0x55e38fed4e60 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28e409f1a0 con 0x55e38fed4e60 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f28dc004700 con 0x55e38fed4e60 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 10 client.25532 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 10 client.25532 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 20 client.25532 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38ff3ad30 0x55e38ff3d150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 4 client.25532 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.692+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55e38fc6ad70 con 0x55e38ff3ad30 2024-04-06T14:07:44.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.693+0000 7f28f216d700 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38ff3ad30 0x55e38ff3d150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.693+0000 7f28d1ffb700 10 client.25532 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.693+0000 7f28f216d700 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38ff3ad30 0x55e38ff3d150 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a8539409ececa2be server_cookie=e45620bf13aaf090 in_seq=0 out_seq=0 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.694+0000 7f28d1ffb700 1 -- 172.21.15.90:0/2213801118 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55e38fc6ad70 con 0x55e38ff3ad30 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.694+0000 7f28d1ffb700 10 client.25532 handle_command_reply: tid=0 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.694+0000 7f2900585b80 1 client.25532 shutdown 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 msgr2=0x55e38ff38df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38ff38df0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f28dc018d20 tx=0x7f28dc019fc0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 shutdown_connections 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e38fed4e60 0x55e38ff38df0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e38ff3ad30 0x55e38ff3d150 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e38fecb120 0x55e38feda750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 --2- 172.21.15.90:0/2213801118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e38fed2870 0x55e38fedac90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f28d1ffb700 0 client.25532 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 >> 172.21.15.90:0/2213801118 conn(0x55e38fe29720 msgr2=0x55e38feca080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 shutdown_connections 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 1 -- 172.21.15.90:0/2213801118 wait complete. 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.695+0000 7f2900585b80 20 client.25532 trim_cache size 0 max 16384 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "b7c5db09-ba2e-4429-b348-86fe6bad403f", 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:07:44.700 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:44.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 msgr2=0x7f28d40a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 0x7f28d40a37d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f28e0006fd0 tx=0x7f28e0008040 comp rx=0 tx=0).stop 2024-04-06T14:07:44.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 msgr2=0x7f28ec114680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f28e4006010 tx=0x7f28e40040b0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 shutdown_connections 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f28d40a1310 0x7f28d40a37d0 secure :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f28e0006fd0 tx=0x7f28e0008040 comp rx=0 tx=0).stop 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28ec111c90 0x7f28ec112140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f28ec1088b0 0x7f28ec114680 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 --2- 172.21.15.90:0/642337624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28ec10cf30 0x7f28ec111750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.696+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 >> 172.21.15.90:0/642337624 conn(0x7f28ec06be00 msgr2=0x7f28ec070940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.697+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 shutdown_connections 2024-04-06T14:07:44.702 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.697+0000 7f28f43d1700 1 -- 172.21.15.90:0/642337624 wait complete. 2024-04-06T14:07:44.713 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:07:44.713 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:07:44.742 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:44 smithi090 ceph-mon[43420]: pgmap v73: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:44.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:44 smithi134 ceph-mon[47549]: pgmap v73: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:44.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- 172.21.15.90:0/1560994520 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac40ffbe0 msgr2=0x7fdac410dbb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/1560994520 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac40ffbe0 0x7fdac410dbb0 secure :-1 s=READY pgs=504 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00b0a0 tx=0x7fdabc00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- 172.21.15.90:0/1560994520 shutdown_connections 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/1560994520 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac40ffbe0 0x7fdac410dbb0 unknown :-1 s=CLOSED pgs=504 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/1560994520 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac4105d00 0x7fdac41060e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/1560994520 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac40ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- 172.21.15.90:0/1560994520 >> 172.21.15.90:0/1560994520 conn(0x7fdac40761b0 msgr2=0x7fdac40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- 172.21.15.90:0/1560994520 shutdown_connections 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- 172.21.15.90:0/1560994520 wait complete. 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 Processor -- start 2024-04-06T14:07:44.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.897+0000 7fdacb2f6700 1 -- start start 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 0x7fdac4110910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac4105d00 0x7fdac4114c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdac4115260 con 0x7fdac4105d00 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdac41153d0 con 0x7fdac40ff220 2024-04-06T14:07:44.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdacb2f6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdac4115590 con 0x7fdac40ffbe0 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdac9092700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdac9092700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60088/0 (socket says 172.21.15.90:60088) 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdac9092700 1 -- 172.21.15.90:0/95405353 learned_addr learned my addr 172.21.15.90:0/95405353 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.898+0000 7fdac9893700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac4105d00 0x7fdac4114c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 -- 172.21.15.90:0/95405353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 msgr2=0x7fdac4110910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac8891700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 0x7fdac4110910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 0x7fdac4110910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 -- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac4105d00 msgr2=0x7fdac4114c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac4105d00 0x7fdac4114c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 -- 172.21.15.90:0/95405353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdabc009d00 con 0x7fdac40ff220 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac8891700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 0x7fdac4110910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:07:44.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.899+0000 7fdac9092700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fdac000e9a0 tx=0x7fdac000ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.900+0000 7fdaba7fc700 1 -- 172.21.15.90:0/95405353 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdac0009790 con 0x7fdac40ff220 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.900+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdac4115870 con 0x7fdac40ff220 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.900+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdac406a9b0 con 0x7fdac40ff220 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.900+0000 7fdaba7fc700 1 -- 172.21.15.90:0/95405353 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdac000cbd0 con 0x7fdac40ff220 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.900+0000 7fdaba7fc700 1 -- 172.21.15.90:0/95405353 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdac0018680 con 0x7fdac40ff220 2024-04-06T14:07:44.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.902+0000 7fdaba7fc700 1 -- 172.21.15.90:0/95405353 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fdac000cd40 con 0x7fdac40ff220 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.904+0000 7fdad74aab80 1 Processor -- start 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.904+0000 7fdad74aab80 1 -- start start 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.904+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 0x55ddb3226b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 0x55ddb32308a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb32331d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdad74aab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb317e990 con 0x55ddb32270a0 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdad74aab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb317c350 con 0x55ddb3226780 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdad74aab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3177c40 con 0x55ddb3230de0 2024-04-06T14:07:44.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9893700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 0x55ddb3226b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9092700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 0x55ddb32308a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9092700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 0x55ddb32308a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45160/0 (socket says 172.21.15.90:45160) 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9092700 1 -- 172.21.15.90:0/830389235 learned_addr learned my addr 172.21.15.90:0/830389235 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac8891700 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb32331d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdaba7fc700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 0x7fdab00b47e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9893700 1 -- 172.21.15.90:0/830389235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 msgr2=0x55ddb32331d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.905+0000 7fdac9893700 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb32331d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdac9893700 1 -- 172.21.15.90:0/830389235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 msgr2=0x55ddb32308a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdac9893700 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 0x55ddb32308a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdac9893700 1 -- 172.21.15.90:0/830389235 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ddb2ff93b0 con 0x55ddb3226780 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdaba7fc700 1 -- 172.21.15.90:0/95405353 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fdac0014070 con 0x7fdac40ff220 2024-04-06T14:07:44.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdac9893700 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 0x55ddb3226b60 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fdabc01a040 tx=0x7fdabc0092e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2d6f287b83d9981a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdaaf7fe700 1 -- 172.21.15.90:0/830389235 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc00dcd0 con 0x55ddb3226780 2024-04-06T14:07:44.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdaaf7fe700 1 -- 172.21.15.90:0/830389235 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdabc00de30 con 0x55ddb3226780 2024-04-06T14:07:44.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdaaf7fe700 1 -- 172.21.15.90:0/830389235 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc010420 con 0x55ddb3226780 2024-04-06T14:07:44.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.906+0000 7fdac9893700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 0x7fdab00b47e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 -- 172.21.15.90:0/830389235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 msgr2=0x55ddb3226b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 0x55ddb3226b60 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fdabc01a040 tx=0x7fdabc0092e0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdac9893700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 0x7fdab00b47e0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00a000 tx=0x7fdabc0106b0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 -- 172.21.15.90:0/830389235 shutdown_connections 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb32331d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb3226780 0x55ddb3226b60 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 --2- 172.21.15.90:0/830389235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb32270a0 0x55ddb32308a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.907+0000 7fdad74aab80 1 -- 172.21.15.90:0/830389235 >> 172.21.15.90:0/830389235 conn(0x55ddb31856a0 msgr2=0x55ddb3221a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.908+0000 7fdad74aab80 1 -- 172.21.15.90:0/830389235 shutdown_connections 2024-04-06T14:07:44.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.908+0000 7fdad74aab80 1 -- 172.21.15.90:0/830389235 wait complete. 2024-04-06T14:07:44.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 -- start start 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 Processor -- start 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb32306c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322da00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3113d80 con 0x55ddb3230de0 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3113a20 con 0x55ddb32270a0 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb2ffb6a0 con 0x55ddb3226780 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdad74aab80 10 client.0 _ll_register_callbacks cb 0x55ddb311d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdac9893700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdac9893700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45162/0 (socket says 172.21.15.90:45162) 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.909+0000 7fdac9893700 1 -- 172.21.15.90:0/365555209 learned_addr learned my addr 172.21.15.90:0/365555209 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac8891700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322da00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9092700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb32306c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 -- 172.21.15.90:0/365555209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 msgr2=0x55ddb32306c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb32306c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 -- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 msgr2=0x55ddb322da00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322da00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 -- 172.21.15.90:0/365555209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdabc007e70 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac8891700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322da00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdac9893700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00b3f0 tx=0x7fdabc010640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c22f65f762e43612 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdaacff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc0076c0 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fdabc031050 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.910+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ddb3138380 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.911+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdabc0103e0 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.911+0000 7fdad74aab80 10 client.15888 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.911+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ddb2fa5b60 con 0x55ddb3230de0 2024-04-06T14:07:44.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.911+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ddb3138380 con 0x55ddb3230de0 2024-04-06T14:07:44.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fdabc03e070 con 0x55ddb3230de0 2024-04-06T14:07:44.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 10 client.15888 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 10 client.15888 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 20 client.15888 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32404d0 0x55ddb32428f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 4 client.15888 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.912+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ddb3170700 con 0x55ddb32404d0 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.913+0000 7fdac9092700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32404d0 0x55ddb32428f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.914+0000 7fdaacff9700 10 client.15888 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.914+0000 7fdac9092700 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32404d0 0x55ddb32428f0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8052507a36fc006f server_cookie=34058c9d608ff6b0 in_seq=0 out_seq=0 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.914+0000 7fdaacff9700 1 -- 172.21.15.90:0/365555209 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ddb3170700 con 0x55ddb32404d0 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.914+0000 7fdaacff9700 10 client.15888 handle_command_reply: tid=0 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 client.15888 shutdown 2024-04-06T14:07:44.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 msgr2=0x55ddb322df60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 secure :-1 s=READY pgs=505 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00b3f0 tx=0x7fdabc010640 comp rx=0 tx=0).stop 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 shutdown_connections 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb322df60 unknown :-1 s=CLOSED pgs=505 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32404d0 0x55ddb32428f0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb32306c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 --2- 172.21.15.90:0/365555209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322da00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 >> 172.21.15.90:0/365555209 conn(0x55ddb31856a0 msgr2=0x55ddb3220c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdaacff9700 0 client.15888 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 shutdown_connections 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 1 -- 172.21.15.90:0/365555209 wait complete. 2024-04-06T14:07:44.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.915+0000 7fdad74aab80 20 client.15888 trim_cache size 0 max 16384 2024-04-06T14:07:44.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 Processor -- start 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 -- start start 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb322c310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3170730 con 0x55ddb3230de0 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb31383b0 con 0x55ddb32270a0 2024-04-06T14:07:44.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.943+0000 7fdad74aab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb322cd90 con 0x55ddb3226780 2024-04-06T14:07:44.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac8891700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45166/0 (socket says 172.21.15.90:45166) 2024-04-06T14:07:44.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 -- 172.21.15.90:0/1499998220 learned_addr learned my addr 172.21.15.90:0/1499998220 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac8891700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60132/0 (socket says 172.21.15.90:60132) 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9092700 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb322c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 -- 172.21.15.90:0/1499998220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 msgr2=0x55ddb322c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb322c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 -- 172.21.15.90:0/1499998220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 msgr2=0x55ddb322c850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 -- 172.21.15.90:0/1499998220 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ddb313dea0 con 0x55ddb3230de0 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac8891700 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdac9893700 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00b780 tx=0x7fdabc022ab0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b04d094ae208d2b6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.944+0000 7fdaaffff700 1 -- 172.21.15.90:0/1499998220 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc0103e0 con 0x55ddb3230de0 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.945+0000 7fdaaffff700 1 -- 172.21.15.90:0/1499998220 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdabc0076c0 con 0x55ddb3230de0 2024-04-06T14:07:44.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.945+0000 7fdaaffff700 1 -- 172.21.15.90:0/1499998220 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdabc022860 con 0x55ddb3230de0 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.945+0000 7fdad74aab80 1 -- 172.21.15.90:0/1499998220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 msgr2=0x55ddb32213a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.945+0000 7fdad74aab80 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 secure :-1 s=READY pgs=506 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00b780 tx=0x7fdabc022ab0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.946+0000 7fdad74aab80 1 -- 172.21.15.90:0/1499998220 shutdown_connections 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.946+0000 7fdad74aab80 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3230de0 0x55ddb32213a0 unknown :-1 s=CLOSED pgs=506 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.946+0000 7fdad74aab80 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3226780 0x55ddb322c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.946+0000 7fdad74aab80 1 --2- 172.21.15.90:0/1499998220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb322c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.946+0000 7fdad74aab80 1 -- 172.21.15.90:0/1499998220 >> 172.21.15.90:0/1499998220 conn(0x55ddb31856a0 msgr2=0x55ddb3228d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.947+0000 7fdad74aab80 1 -- 172.21.15.90:0/1499998220 shutdown_connections 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.947+0000 7fdad74aab80 1 -- 172.21.15.90:0/1499998220 wait complete. 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.947+0000 7fdad74aab80 1 -- start start 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.947+0000 7fdad74aab80 1 Processor -- start 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 0x55ddb3235c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb32361a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb3293300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb32369b0 con 0x55ddb3226780 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3293870 con 0x55ddb32270a0 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ddb3293a30 con 0x55ddb3230de0 2024-04-06T14:07:44.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdad74aab80 10 client.0 _ll_register_callbacks cb 0x55ddb311d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac9092700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 0x55ddb3235c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac9092700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 0x55ddb3235c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:45174/0 (socket says 172.21.15.90:45174) 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac9092700 1 -- 172.21.15.90:0/2202972647 learned_addr learned my addr 172.21.15.90:0/2202972647 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb32361a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac9893700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb3293300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 -- 172.21.15.90:0/2202972647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 msgr2=0x55ddb3293300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb3293300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 -- 172.21.15.90:0/2202972647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 msgr2=0x55ddb3235c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 0x55ddb3235c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.948+0000 7fdac8891700 1 -- 172.21.15.90:0/2202972647 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdab4004340 con 0x55ddb32270a0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdac8891700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb32361a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x55ddb32286e0 tx=0x7fdab40098a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ed9c3f0ba370e1b0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:07:44.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdab4018070 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fdab4012450 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fdabc00de30 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdab4003f20 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 10 client.25550 fetch_fsmap learned FSMap version 24 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ddb2fa5b60 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdab4029080 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fdab4011650 con 0x55ddb32270a0 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 10 client.25550 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 10 client.25550 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 20 client.25550 populate_metadata read hostname 'smithi090' 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32952c0 0x55ddb32976e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 4 client.25550 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:07:44.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.949+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ddb2fc6cf0 con 0x55ddb32952c0 2024-04-06T14:07:44.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.951+0000 7fdac9092700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32952c0 0x55ddb32976e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:07:44.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.952+0000 7fdaacff9700 10 client.25550 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.952+0000 7fdac9092700 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32952c0 0x55ddb32976e0 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=32297872b8a060ff server_cookie=31399597b2327a15 in_seq=0 out_seq=0 2024-04-06T14:07:44.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.953+0000 7fdaacff9700 1 -- 172.21.15.90:0/2202972647 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55ddb2fc6cf0 con 0x55ddb32952c0 2024-04-06T14:07:44.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.953+0000 7fdaacff9700 10 client.25550 handle_command_reply: tid=0 2024-04-06T14:07:44.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.953+0000 7fdad74aab80 1 client.25550 shutdown 2024-04-06T14:07:44.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 msgr2=0x55ddb32361a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb32361a0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x55ddb32286e0 tx=0x7fdab40098a0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 shutdown_connections 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ddb32952c0 0x55ddb32976e0 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ddb3230de0 0x55ddb3293300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdaacff9700 0 client.25550 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ddb3226780 0x55ddb3235c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 --2- 172.21.15.90:0/2202972647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ddb32270a0 0x55ddb32361a0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 >> 172.21.15.90:0/2202972647 conn(0x55ddb31856a0 msgr2=0x55ddb322c250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 shutdown_connections 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 1 -- 172.21.15.90:0/2202972647 wait complete. 2024-04-06T14:07:44.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.954+0000 7fdad74aab80 20 client.25550 trim_cache size 0 max 16384 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "b7c5db09-ba2e-4429-b348-86fe6bad403f": { 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "b7c5db09-ba2e-4429-b348-86fe6bad403f", 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:07:44.956 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:07:44.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 msgr2=0x7fdab00b47e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 0x7fdab00b47e0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fdabc00a000 tx=0x7fdabc0106b0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 msgr2=0x7fdac41103d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fdac000e9a0 tx=0x7fdac000ed60 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 shutdown_connections 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdab00b2320 0x7fdab00b47e0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdac4105d00 0x7fdac4114c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdac40ff220 0x7fdac41103d0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 --2- 172.21.15.90:0/95405353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdac40ffbe0 0x7fdac4110910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 >> 172.21.15.90:0/95405353 conn(0x7fdac40761b0 msgr2=0x7fdac4103d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 shutdown_connections 2024-04-06T14:07:44.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:07:44.956+0000 7fdacb2f6700 1 -- 172.21.15.90:0/95405353 wait complete. 2024-04-06T14:07:44.976 INFO:tasks.cephfs.filesystem:scrub status for tag:b7c5db09-ba2e-4429-b348-86fe6bad403f - {'path': '/', 'tag': 'b7c5db09-ba2e-4429-b348-86fe6bad403f', 'options': 'recursive,force'} 2024-04-06T14:07:44.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:44 smithi105 ceph-mon[46969]: pgmap v73: 97 pgs: 97 active+clean; 3.6 GiB data, 4.6 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 17 MiB/s wr, 815 op/s 2024-04-06T14:07:45.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:45 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:07:45.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:45 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:45.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:45 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:07:45.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:45 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:45.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:45 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:07:45.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:45 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:07:45.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:45 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:45.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:45 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:07:45.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:45 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:45.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:45 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:07:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:45 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:07:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:45 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:45 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:07:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:45 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:07:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:45 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:07:46.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:46 smithi134 ceph-mon[47549]: pgmap v74: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 29 MiB/s wr, 1.17k op/s 2024-04-06T14:07:46.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:46 smithi105 ceph-mon[46969]: pgmap v74: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 29 MiB/s wr, 1.17k op/s 2024-04-06T14:07:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:46 smithi090 ceph-mon[43420]: pgmap v74: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 29 MiB/s wr, 1.17k op/s 2024-04-06T14:07:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 120.0 s, 1970.7 tps, lat 49.488 ms stddev 81.109 2024-04-06T14:07:48.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:48 smithi134 ceph-mon[47549]: pgmap v75: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 22 MiB/s wr, 825 op/s 2024-04-06T14:07:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:48 smithi105 ceph-mon[46969]: pgmap v75: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 22 MiB/s wr, 825 op/s 2024-04-06T14:07:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:48 smithi090 ceph-mon[43420]: pgmap v75: 97 pgs: 97 active+clean; 3.7 GiB data, 4.7 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 22 MiB/s wr, 825 op/s 2024-04-06T14:07:49.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:49 smithi105 ceph-mon[46969]: pgmap v76: 97 pgs: 97 active+clean; 3.8 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 36 MiB/s wr, 1.50k op/s 2024-04-06T14:07:50.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:49 smithi090 ceph-mon[43420]: pgmap v76: 97 pgs: 97 active+clean; 3.8 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 36 MiB/s wr, 1.50k op/s 2024-04-06T14:07:50.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:49 smithi134 ceph-mon[47549]: pgmap v76: 97 pgs: 97 active+clean; 3.8 GiB data, 4.8 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 36 MiB/s wr, 1.50k op/s 2024-04-06T14:07:51.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:51 smithi105 ceph-mon[46969]: pgmap v77: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 43 MiB/s wr, 1.76k op/s 2024-04-06T14:07:52.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:51 smithi090 ceph-mon[43420]: pgmap v77: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 43 MiB/s wr, 1.76k op/s 2024-04-06T14:07:52.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:51 smithi134 ceph-mon[47549]: pgmap v77: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 43 MiB/s wr, 1.76k op/s 2024-04-06T14:07:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 125.0 s, 2519.0 tps, lat 40.812 ms stddev 88.678 2024-04-06T14:07:53.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:53 smithi105 ceph-mon[46969]: pgmap v78: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 40 MiB/s wr, 1.72k op/s 2024-04-06T14:07:54.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:53 smithi090 ceph-mon[43420]: pgmap v78: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 40 MiB/s wr, 1.72k op/s 2024-04-06T14:07:54.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:53 smithi134 ceph-mon[47549]: pgmap v78: 97 pgs: 97 active+clean; 3.8 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 40 MiB/s wr, 1.72k op/s 2024-04-06T14:07:55.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:55 smithi105 ceph-mon[46969]: pgmap v79: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 53 MiB/s wr, 2.22k op/s 2024-04-06T14:07:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:55 smithi090 ceph-mon[43420]: pgmap v79: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 53 MiB/s wr, 2.22k op/s 2024-04-06T14:07:56.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:55 smithi134 ceph-mon[47549]: pgmap v79: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 53 MiB/s wr, 2.22k op/s 2024-04-06T14:07:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 130.0 s, 3643.8 tps, lat 27.454 ms stddev 28.539 2024-04-06T14:07:58.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:58 smithi134 ceph-mon[47549]: pgmap v80: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 41 MiB/s wr, 1.86k op/s 2024-04-06T14:07:58.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:58 smithi105 ceph-mon[46969]: pgmap v80: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 41 MiB/s wr, 1.86k op/s 2024-04-06T14:07:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:58 smithi090 ceph-mon[43420]: pgmap v80: 97 pgs: 97 active+clean; 3.9 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 41 MiB/s wr, 1.86k op/s 2024-04-06T14:07:59.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:07:59 smithi105 ceph-mon[46969]: pgmap v81: 97 pgs: 97 active+clean; 3.9 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 45 MiB/s wr, 2.02k op/s 2024-04-06T14:08:00.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:07:59 smithi090 ceph-mon[43420]: pgmap v81: 97 pgs: 97 active+clean; 3.9 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 45 MiB/s wr, 2.02k op/s 2024-04-06T14:08:00.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:07:59 smithi134 ceph-mon[47549]: pgmap v81: 97 pgs: 97 active+clean; 3.9 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 45 MiB/s wr, 2.02k op/s 2024-04-06T14:08:01.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:01 smithi105 ceph-mon[46969]: pgmap v82: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 36 MiB/s wr, 1.52k op/s 2024-04-06T14:08:02.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:01 smithi090 ceph-mon[43420]: pgmap v82: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 36 MiB/s wr, 1.52k op/s 2024-04-06T14:08:02.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:01 smithi134 ceph-mon[47549]: pgmap v82: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 36 MiB/s wr, 1.52k op/s 2024-04-06T14:08:03.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 135.0 s, 2336.6 tps, lat 42.636 ms stddev 85.617 2024-04-06T14:08:03.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:03 smithi105 ceph-mon[46969]: pgmap v83: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 21 MiB/s wr, 830 op/s 2024-04-06T14:08:04.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:03 smithi090 ceph-mon[43420]: pgmap v83: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 21 MiB/s wr, 830 op/s 2024-04-06T14:08:04.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:03 smithi134 ceph-mon[47549]: pgmap v83: 97 pgs: 97 active+clean; 4.0 GiB data, 5.1 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 21 MiB/s wr, 830 op/s 2024-04-06T14:08:05.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:05 smithi105 ceph-mon[46969]: pgmap v84: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 47 MiB/s wr, 2.36k op/s 2024-04-06T14:08:06.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:05 smithi090 ceph-mon[43420]: pgmap v84: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 47 MiB/s wr, 2.36k op/s 2024-04-06T14:08:06.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:05 smithi134 ceph-mon[47549]: pgmap v84: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 47 MiB/s wr, 2.36k op/s 2024-04-06T14:08:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 140.0 s, 2340.4 tps, lat 42.927 ms stddev 54.135 2024-04-06T14:08:08.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:08 smithi134 ceph-mon[47549]: pgmap v85: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 34 MiB/s wr, 1.86k op/s 2024-04-06T14:08:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:08 smithi105 ceph-mon[46969]: pgmap v85: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 34 MiB/s wr, 1.86k op/s 2024-04-06T14:08:09.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:08 smithi090 ceph-mon[43420]: pgmap v85: 97 pgs: 97 active+clean; 4.0 GiB data, 5.3 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 34 MiB/s wr, 1.86k op/s 2024-04-06T14:08:09.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:09 smithi105 ceph-mon[46969]: pgmap v86: 97 pgs: 97 active+clean; 4.1 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 43 MiB/s wr, 2.16k op/s 2024-04-06T14:08:10.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:09 smithi090 ceph-mon[43420]: pgmap v86: 97 pgs: 97 active+clean; 4.1 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 43 MiB/s wr, 2.16k op/s 2024-04-06T14:08:10.147 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:09 smithi134 ceph-mon[47549]: pgmap v86: 97 pgs: 97 active+clean; 4.1 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 43 MiB/s wr, 2.16k op/s 2024-04-06T14:08:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:11 smithi105 ceph-mon[46969]: pgmap v87: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 44 MiB/s wr, 2.25k op/s 2024-04-06T14:08:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:11 smithi090 ceph-mon[43420]: pgmap v87: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 44 MiB/s wr, 2.25k op/s 2024-04-06T14:08:12.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:11 smithi134 ceph-mon[47549]: pgmap v87: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 44 MiB/s wr, 2.25k op/s 2024-04-06T14:08:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 145.0 s, 2435.0 tps, lat 40.014 ms stddev 43.417 2024-04-06T14:08:13.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:13 smithi105 ceph-mon[46969]: pgmap v88: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 40 MiB/s wr, 2.09k op/s 2024-04-06T14:08:14.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:13 smithi090 ceph-mon[43420]: pgmap v88: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 40 MiB/s wr, 2.09k op/s 2024-04-06T14:08:14.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:13 smithi134 ceph-mon[47549]: pgmap v88: 97 pgs: 97 active+clean; 4.2 GiB data, 5.5 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 40 MiB/s wr, 2.09k op/s 2024-04-06T14:08:14.977 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:08:14.978 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:08:15.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.163+0000 7f709f4fa700 1 -- 172.21.15.90:0/272460619 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 msgr2=0x7f7098100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.163+0000 7f709f4fa700 1 --2- 172.21.15.90:0/272460619 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098100250 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f70940098f0 tx=0x7f7094009c00 comp rx=0 tx=0).stop 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.163+0000 7f709f4fa700 1 -- 172.21.15.90:0/272460619 shutdown_connections 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 --2- 172.21.15.90:0/272460619 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7098104f70 0x7f709810caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 --2- 172.21.15.90:0/272460619 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098100250 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 --2- 172.21.15.90:0/272460619 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098100820 0x7f7098104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 -- 172.21.15.90:0/272460619 >> 172.21.15.90:0/272460619 conn(0x7f7098076d50 msgr2=0x7f7098077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 -- 172.21.15.90:0/272460619 shutdown_connections 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 -- 172.21.15.90:0/272460619 wait complete. 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 Processor -- start 2024-04-06T14:08:15.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.164+0000 7f709f4fa700 1 -- start start 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098195e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 0x7f7098196380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70981a5a00 con 0x7f7098104f70 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70981a5b70 con 0x7f70980ffe70 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709f4fa700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70981a5d30 con 0x7f70981968c0 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:32820/0 (socket says 172.21.15.90:32820) 2024-04-06T14:08:15.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 -- 172.21.15.90:0/1011192845 learned_addr learned my addr 172.21.15.90:0/1011192845 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709d296700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098195e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709ca95700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 0x7f7098196380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 -- 172.21.15.90:0/1011192845 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 msgr2=0x7f7098195e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098195e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 -- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 msgr2=0x7f7098196380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 0x7f7098196380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 -- 172.21.15.90:0/1011192845 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70880095f0 con 0x7f70981968c0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709ca95700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 0x7f7098196380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709d296700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098195e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.165+0000 7f709da97700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f709000ef30 tx=0x7f709000c450 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.166+0000 7f708e7fc700 1 -- 172.21.15.90:0/1011192845 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090018070 con 0x7f70981968c0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.166+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7094009520 con 0x7f70981968c0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.166+0000 7f708e7fc700 1 -- 172.21.15.90:0/1011192845 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f709000f040 con 0x7f70981968c0 2024-04-06T14:08:15.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.166+0000 7f708e7fc700 1 -- 172.21.15.90:0/1011192845 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7090003dc0 con 0x7f70981968c0 2024-04-06T14:08:15.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.166+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70981a6300 con 0x7f70981968c0 2024-04-06T14:08:15.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f708e7fc700 1 -- 172.21.15.90:0/1011192845 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f709000caa0 con 0x7f70981968c0 2024-04-06T14:08:15.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 Processor -- start 2024-04-06T14:08:15.170 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 -- start start 2024-04-06T14:08:15.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15e0370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f5450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd152f990 con 0x562fd15dff90 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd152d350 con 0x562fd15f3060 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd1528c40 con 0x562fd15e08b0 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f708e7fc700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 0x7f70840a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f708e7fc700 1 -- 172.21.15.90:0/1011192845 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f70900cc900 con 0x7f70981968c0 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f709d296700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f709d296700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:32830/0 (socket says 172.21.15.90:32830) 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.171+0000 7f709d296700 1 -- 172.21.15.90:0/150455688 learned_addr learned my addr 172.21.15.90:0/150455688 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709da97700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 0x7f70840a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709ca95700 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f5450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709ca95700 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15e0370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 -- 172.21.15.90:0/150455688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 msgr2=0x562fd15f5450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f5450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 -- 172.21.15.90:0/150455688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 msgr2=0x562fd15e0370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15e0370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 -- 172.21.15.90:0/150455688 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562fd13aa3b0 con 0x562fd15e08b0 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709da97700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 0x7f70840a3850 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f7090009f40 tx=0x7f7090009ed0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f709d296700 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f70940098f0 tx=0x7f709400bef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a4ab9e5f6bc23c2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f70837fe700 1 -- 172.21.15.90:0/150455688 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f709401c040 con 0x562fd15e08b0 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f70837fe700 1 -- 172.21.15.90:0/150455688 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f709400f920 con 0x562fd15e08b0 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.172+0000 7f70837fe700 1 -- 172.21.15.90:0/150455688 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7094005240 con 0x562fd15e08b0 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.173+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/150455688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 msgr2=0x562fd15dc7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.173+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f70940098f0 tx=0x7f709400bef0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/150455688 shutdown_connections 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f5450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15e0370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/150455688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15dc7c0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/150455688 >> 172.21.15.90:0/150455688 conn(0x562fd15366a0 msgr2=0x562fd15d2a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/150455688 shutdown_connections 2024-04-06T14:08:15.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/150455688 wait complete. 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 -- start start 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.175+0000 7f70ab6aeb80 1 Processor -- start 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15dc770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15da0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd14c4d80 con 0x562fd15dff90 2024-04-06T14:08:15.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd14c4a20 con 0x562fd15e08b0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd13ac6a0 con 0x562fd15f3060 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f70ab6aeb80 10 client.0 _ll_register_callbacks cb 0x562fd14ce470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38400/0 (socket says 172.21.15.90:38400) 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 -- 172.21.15.90:0/586768910 learned_addr learned my addr 172.21.15.90:0/586768910 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 -- 172.21.15.90:0/586768910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 msgr2=0x562fd15da0e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15da0e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 -- 172.21.15.90:0/586768910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 msgr2=0x562fd15dc770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15dc770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.176+0000 7f709d296700 1 -- 172.21.15.90:0/586768910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f709810d360 con 0x562fd15e08b0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f709d296700 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f709400ff40 tx=0x7f709400b6e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b32d39106f48b540 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f7080ff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f709401c070 con 0x562fd15e08b0 2024-04-06T14:08:15.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f70940050b0 con 0x562fd15e08b0 2024-04-06T14:08:15.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562fd14e9380 con 0x562fd15e08b0 2024-04-06T14:08:15.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f709400b7f0 con 0x562fd15e08b0 2024-04-06T14:08:15.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f70ab6aeb80 10 client.25562 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:15.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562fd1356b60 con 0x562fd15e08b0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.177+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x562fd14e9380 con 0x562fd15e08b0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7094033070 con 0x562fd15e08b0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 10 client.25562 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 10 client.25562 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 20 client.25562 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd16024d0 0x562fd16048f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 4 client.25562 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562fd1521700 con 0x562fd16024d0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.178+0000 7f709ca95700 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd16024d0 0x562fd16048f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.179+0000 7f709ca95700 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd16024d0 0x562fd16048f0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a1162f6b6ab0285a server_cookie=e7983100d3152b56 in_seq=0 out_seq=0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.179+0000 7f7080ff9700 10 client.25562 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.179+0000 7f7080ff9700 1 -- 172.21.15.90:0/586768910 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x7f708801e730 con 0x562fd16024d0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.179+0000 7f7080ff9700 10 client.25562 handle_command_reply: tid=0 2024-04-06T14:08:15.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 client.25562 shutdown 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 msgr2=0x562fd15d99f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f709400ff40 tx=0x7f709400b6e0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 shutdown_connections 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd16024d0 0x562fd16048f0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15da0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15dc770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/586768910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15d99f0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f7080ff9700 0 client.25562 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:15.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 >> 172.21.15.90:0/586768910 conn(0x562fd15366a0 msgr2=0x562fd15d1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 shutdown_connections 2024-04-06T14:08:15.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.180+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/586768910 wait complete. 2024-04-06T14:08:15.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.181+0000 7f70ab6aeb80 20 client.25562 trim_cache size 0 max 16384 2024-04-06T14:08:15.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 Processor -- start 2024-04-06T14:08:15.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 -- start start 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15d2b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15f9a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15fe8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15e1270 con 0x562fd15dff90 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15dae80 con 0x562fd15e08b0 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15e1910 con 0x562fd15f3060 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f709d296700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15f9a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f709ca95700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15d2b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f709ca95700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15d2b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34858/0 (socket says 172.21.15.90:34858) 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f709ca95700 1 -- 172.21.15.90:0/1518477502 learned_addr learned my addr 172.21.15.90:0/1518477502 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.207+0000 7f709ca95700 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15fe8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 -- 172.21.15.90:0/1518477502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 msgr2=0x562fd15fe8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15fe8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 -- 172.21.15.90:0/1518477502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 msgr2=0x562fd15d2b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15d2b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 -- 172.21.15.90:0/1518477502 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562fd14eeea0 con 0x562fd15e08b0 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f709d296700 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15f9a60 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f7094000c00 tx=0x7f7094003de0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=79a689dce2b1b5e9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f7083fff700 1 -- 172.21.15.90:0/1518477502 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f709401c040 con 0x562fd15e08b0 2024-04-06T14:08:15.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f7083fff700 1 -- 172.21.15.90:0/1518477502 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f709400b560 con 0x562fd15e08b0 2024-04-06T14:08:15.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.208+0000 7f7083fff700 1 -- 172.21.15.90:0/1518477502 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7094005300 con 0x562fd15e08b0 2024-04-06T14:08:15.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.209+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/1518477502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 msgr2=0x562fd15f9a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.209+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15f9a60 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f7094000c00 tx=0x7f7094003de0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.209+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/1518477502 shutdown_connections 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.209+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15f3060 0x562fd15fe8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.209+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15d2b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/1518477502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15e08b0 0x562fd15f9a60 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/1518477502 >> 172.21.15.90:0/1518477502 conn(0x562fd15366a0 msgr2=0x562fd15fe1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/1518477502 shutdown_connections 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/1518477502 wait complete. 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 -- start start 2024-04-06T14:08:15.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.210+0000 7f70ab6aeb80 1 Processor -- start 2024-04-06T14:08:15.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15f7e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f83b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15f8920 con 0x562fd15dff90 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15f8a90 con 0x562fd15f3060 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562fd15f8c50 con 0x562fd15e08b0 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f70ab6aeb80 10 client.0 _ll_register_callbacks cb 0x562fd14ce470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34870/0 (socket says 172.21.15.90:34870) 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 -- 172.21.15.90:0/2749432824 learned_addr learned my addr 172.21.15.90:0/2749432824 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f83b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709d296700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15f7e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 -- 172.21.15.90:0/2749432824 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 msgr2=0x562fd15f7e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15f7e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 -- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 msgr2=0x562fd15f83b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f83b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.211+0000 7f709ca95700 1 -- 172.21.15.90:0/2749432824 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562fd1356b60 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f709ca95700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f7088005fd0 tx=0x7f7088009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d42f584d8ffbfc4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f7080ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7088004a40 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f7088004ba0 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562fd1377cf0 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f70880160d0 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f70ab6aeb80 10 client.15906 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562fd13aecd0 con 0x562fd15dff90 2024-04-06T14:08:15.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.212+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7088032020 con 0x562fd15dff90 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7088011750 con 0x562fd15dff90 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 10 client.15906 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 10 client.15906 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 20 client.15906 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd1668800 0x562fd166ac20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 4 client.15906 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.213+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x562fd13acce0 con 0x562fd1668800 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f709d296700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd1668800 0x562fd166ac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f709d296700 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd1668800 0x562fd166ac20 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3d7d00393d403f7e server_cookie=5bdcfd5ea38a3f4a in_seq=0 out_seq=0 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f7080ff9700 10 client.15906 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f7080ff9700 1 -- 172.21.15.90:0/2749432824 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x562fd13acce0 con 0x562fd1668800 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f7080ff9700 10 client.15906 handle_command_reply: tid=0 2024-04-06T14:08:15.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f70ab6aeb80 1 client.15906 shutdown 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 msgr2=0x562fd15f9860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.214+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 secure :-1 s=READY pgs=507 cs=0 l=1 rev1=1 crypto rx=0x7f7088005fd0 tx=0x7f7088009310 comp rx=0 tx=0).stop 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 shutdown_connections 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562fd15f3060 0x562fd15f83b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562fd15dff90 0x562fd15f9860 unknown :-1 s=CLOSED pgs=507 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562fd1668800 0x562fd166ac20 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 --2- 172.21.15.90:0/2749432824 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562fd15e08b0 0x562fd15f7e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.215+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 >> 172.21.15.90:0/2749432824 conn(0x562fd15366a0 msgr2=0x562fd15d1390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f7080ff9700 0 client.15906 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 shutdown_connections 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f70ab6aeb80 1 -- 172.21.15.90:0/2749432824 wait complete. 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f70ab6aeb80 20 client.15906 trim_cache size 0 max 16384 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:08:15.218 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:08:15.219 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:15.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 msgr2=0x7f70840a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 0x7f70840a3850 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f7090009f40 tx=0x7f7090009ed0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.217+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 msgr2=0x7f70981a54c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f709000ef30 tx=0x7f709000c450 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 shutdown_connections 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70840a1390 0x7f70840a3850 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70981968c0 0x7f70981a54c0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70980ffe70 0x7f7098195e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 --2- 172.21.15.90:0/1011192845 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7098104f70 0x7f7098196380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 >> 172.21.15.90:0/1011192845 conn(0x7f7098076d50 msgr2=0x7f70980fe060 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 shutdown_connections 2024-04-06T14:08:15.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.218+0000 7f709f4fa700 1 -- 172.21.15.90:0/1011192845 wait complete. 2024-04-06T14:08:15.231 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:08:15.232 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 -- 172.21.15.90:0/758184103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b24108ec0 msgr2=0x7f1b24109340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/758184103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b24108ec0 0x7f1b24109340 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f1b1400b970 tx=0x7f1b1400bc80 comp rx=0 tx=0).stop 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 -- 172.21.15.90:0/758184103 shutdown_connections 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/758184103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24109880 0x7f1b24115290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/758184103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b2410e5f0 0x7f1b2410e9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/758184103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b24108ec0 0x7f1b24109340 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 -- 172.21.15.90:0/758184103 >> 172.21.15.90:0/758184103 conn(0x7f1b2406c180 msgr2=0x7f1b2406c590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 -- 172.21.15.90:0/758184103 shutdown_connections 2024-04-06T14:08:15.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.418+0000 7f1b2b32e700 1 -- 172.21.15.90:0/758184103 wait complete. 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 Processor -- start 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 -- start start 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 0x7f1b241a3500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b24109880 0x7f1b241a3a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 0x7f1b2419d680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b2419dce0 con 0x7f1b24109880 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b2419de50 con 0x7f1b24108ec0 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.419+0000 7f1b2b32e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b2419e010 con 0x7f1b2410e5f0 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b288c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b24109880 0x7f1b241a3a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b298cb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 0x7f1b2419d680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b298cb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 0x7f1b2419d680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:32866/0 (socket says 172.21.15.90:32866) 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 0x7f1b241a3500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b298cb700 1 -- 172.21.15.90:0/29259475 learned_addr learned my addr 172.21.15.90:0/29259475 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 -- 172.21.15.90:0/29259475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 msgr2=0x7f1b2419d680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 0x7f1b2419d680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 -- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b24109880 msgr2=0x7f1b241a3a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b24109880 0x7f1b241a3a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 -- 172.21.15.90:0/29259475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b1400b5a0 con 0x7f1b24108ec0 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b290ca700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 0x7f1b241a3500 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f1b2000d950 tx=0x7f1b2000dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.420+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b2000f890 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.421+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1b20010460 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.421+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1b2000e610 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.421+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b2419e2f0 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.421+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b241aa4f0 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.422+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b241109e0 con 0x7f1b24108ec0 2024-04-06T14:08:15.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.423+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1b200105d0 con 0x7f1b24108ec0 2024-04-06T14:08:15.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.424+0000 7f1b1a7fc700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 0x7f1b100ac0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:15.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.424+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1b200cc900 con 0x7f1b24108ec0 2024-04-06T14:08:15.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.424+0000 7f1b288c9700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 0x7f1b100ac0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:15.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.425+0000 7f1b288c9700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 0x7f1b100ac0a0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f1b14009230 tx=0x7f1b1400d040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:15.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.427+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f1b20095de0 con 0x7f1b24108ec0 2024-04-06T14:08:15.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.795+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1b24072b50 con 0x7f1b24108ec0 2024-04-06T14:08:15.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.796+0000 7f1b1a7fc700 1 -- 172.21.15.90:0/29259475 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f1b20020070 con 0x7f1b24108ec0 2024-04-06T14:08:15.797 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:08:15.797 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:08:15.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.801+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 msgr2=0x7f1b100ac0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.801+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 0x7f1b100ac0a0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f1b14009230 tx=0x7f1b1400d040 comp rx=0 tx=0).stop 2024-04-06T14:08:15.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.801+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 msgr2=0x7f1b241a3500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:15.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.801+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 0x7f1b241a3500 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f1b2000d950 tx=0x7f1b2000dc60 comp rx=0 tx=0).stop 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 shutdown_connections 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1b100a9be0 0x7f1b100ac0a0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1b2410e5f0 0x7f1b2419d680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1b24108ec0 0x7f1b241a3500 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 --2- 172.21.15.90:0/29259475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1b24109880 0x7f1b241a3a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 >> 172.21.15.90:0/29259475 conn(0x7f1b2406c180 msgr2=0x7f1b2410ae70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 shutdown_connections 2024-04-06T14:08:15.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:15.802+0000 7f1b2b32e700 1 -- 172.21.15.90:0/29259475 wait complete. 2024-04-06T14:08:15.803 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:08:15.817 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:15.817 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:08:15.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:15 smithi105 ceph-mon[46969]: pgmap v89: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 3.66k op/s 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 -- 172.21.15.90:0/3401255531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06cf60 msgr2=0x7ff35c119f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 --2- 172.21.15.90:0/3401255531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06cf60 0x7ff35c119f10 secure :-1 s=READY pgs=508 cs=0 l=1 rev1=1 crypto rx=0x7ff34c009990 tx=0x7ff34c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 -- 172.21.15.90:0/3401255531 shutdown_connections 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 --2- 172.21.15.90:0/3401255531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c11a450 0x7ff35c11c840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 --2- 172.21.15.90:0/3401255531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c06c640 0x7ff35c06ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 --2- 172.21.15.90:0/3401255531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06cf60 0x7ff35c119f10 unknown :-1 s=CLOSED pgs=508 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 -- 172.21.15.90:0/3401255531 >> 172.21.15.90:0/3401255531 conn(0x7ff35c06bd10 msgr2=0x7ff35c109dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 -- 172.21.15.90:0/3401255531 shutdown_connections 2024-04-06T14:08:16.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.009+0000 7ff362874700 1 -- 172.21.15.90:0/3401255531 wait complete. 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.010+0000 7ff362874700 1 Processor -- start 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.010+0000 7ff362874700 1 -- start start 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 0x7ff35c119990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c06cf60 0x7ff35c119ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 0x7ff35c113c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff35c1143a0 con 0x7ff35c06c640 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff35c114510 con 0x7ff35c06cf60 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff362874700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff35c1146d0 con 0x7ff35c11a450 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff35bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 0x7ff35c119990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff35bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 0x7ff35c119990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34894/0 (socket says 172.21.15.90:34894) 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff35bfff700 1 -- 172.21.15.90:0/2795229057 learned_addr learned my addr 172.21.15.90:0/2795229057 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff360e11700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 0x7ff35c113c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.011+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c06cf60 0x7ff35c119ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 -- 172.21.15.90:0/2795229057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c06cf60 msgr2=0x7ff35c119ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c06cf60 0x7ff35c119ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 -- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 msgr2=0x7ff35c119990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 0x7ff35c119990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 -- 172.21.15.90:0/2795229057 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff34c0095f0 con 0x7ff35c11a450 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff360e11700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 0x7ff35c113c10 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7ff35400ef90 tx=0x7ff35400c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.012+0000 7ff3597fa700 1 -- 172.21.15.90:0/2795229057 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff354018070 con 0x7ff35c11a450 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.013+0000 7ff3597fa700 1 -- 172.21.15.90:0/2795229057 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff35400f040 con 0x7ff35c11a450 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.013+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff35c1bb0e0 con 0x7ff35c11a450 2024-04-06T14:08:16.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.013+0000 7ff3597fa700 1 -- 172.21.15.90:0/2795229057 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff35400cb80 con 0x7ff35c11a450 2024-04-06T14:08:16.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.013+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff35c1bb4b0 con 0x7ff35c11a450 2024-04-06T14:08:16.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.016+0000 7ff3597fa700 1 -- 172.21.15.90:0/2795229057 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff354003a00 con 0x7ff35c11a450 2024-04-06T14:08:16.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.017+0000 7ff36ea28b80 1 Processor -- start 2024-04-06T14:08:16.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.018+0000 7ff3597fa700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 0x7ff3440a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.019+0000 7ff3597fa700 1 -- 172.21.15.90:0/2795229057 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3540cc800 con 0x7ff35c11a450 2024-04-06T14:08:16.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.019+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 0x7ff3440a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 -- start start 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e599b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a61d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e4f1990 con 0x559f8e59a0a0 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e4ef350 con 0x559f8e5a3de0 2024-04-06T14:08:16.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.020+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e4eac40 con 0x559f8e599780 2024-04-06T14:08:16.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34910/0 (socket says 172.21.15.90:34910) 2024-04-06T14:08:16.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 -- 172.21.15.90:0/3674513704 learned_addr learned my addr 172.21.15.90:0/3674513704 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff360e11700 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a61d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 0x7ff3440a3750 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7ff35c1151c0 tx=0x7ff34c005060 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35bfff700 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e599b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 -- 172.21.15.90:0/3674513704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 msgr2=0x559f8e599b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e599b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 -- 172.21.15.90:0/3674513704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 msgr2=0x559f8e5a61d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a61d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.021+0000 7ff35b7fe700 1 -- 172.21.15.90:0/3674513704 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559f8e36c3b0 con 0x559f8e59a0a0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.022+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7ff34c0098e0 tx=0x7ff34c028000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af5168c85629a47b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.022+0000 7ff3427fc700 1 -- 172.21.15.90:0/3674513704 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff34c00be90 con 0x559f8e59a0a0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.022+0000 7ff3427fc700 1 -- 172.21.15.90:0/3674513704 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff34c01e040 con 0x559f8e59a0a0 2024-04-06T14:08:16.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.022+0000 7ff3427fc700 1 -- 172.21.15.90:0/3674513704 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff34c02f760 con 0x559f8e59a0a0 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3674513704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 msgr2=0x559f8e5a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 secure :-1 s=READY pgs=509 cs=0 l=1 rev1=1 crypto rx=0x7ff34c0098e0 tx=0x7ff34c028000 comp rx=0 tx=0).stop 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3674513704 shutdown_connections 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a61d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e599b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3674513704 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a38a0 unknown :-1 s=CLOSED pgs=509 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.023+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3674513704 >> 172.21.15.90:0/3674513704 conn(0x559f8e4f86a0 msgr2=0x559f8e594a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.025+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3674513704 shutdown_connections 2024-04-06T14:08:16.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.025+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3674513704 wait complete. 2024-04-06T14:08:16.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.025+0000 7ff36ea28b80 1 -- start start 2024-04-06T14:08:16.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 Processor -- start 2024-04-06T14:08:16.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e5a0c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a1160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e486d80 con 0x559f8e59a0a0 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e486a20 con 0x559f8e5a3de0 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e36e6a0 con 0x559f8e599780 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff36ea28b80 10 client.0 _ll_register_callbacks cb 0x559f8e490470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff360e11700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff360e11700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38478/0 (socket says 172.21.15.90:38478) 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff35bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e5a0c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff360e11700 1 -- 172.21.15.90:0/4148396596 learned_addr learned my addr 172.21.15.90:0/4148396596 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.026+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a1160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff360e11700 1 -- 172.21.15.90:0/4148396596 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 msgr2=0x559f8e5a0c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff360e11700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e5a0c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff360e11700 1 -- 172.21.15.90:0/4148396596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 msgr2=0x559f8e5a1160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff360e11700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a1160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff360e11700 1 -- 172.21.15.90:0/4148396596 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff35c11d300 con 0x559f8e5a3de0 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.027+0000 7ff35bfff700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e5a0c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.028+0000 7ff360e11700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7ff354099610 tx=0x7ff35401b460 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9b37410324400045 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.028+0000 7ff33b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.028+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff35409d740 con 0x559f8e5a3de0 2024-04-06T14:08:16.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.028+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3540a1d00 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.028+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559f8e4ab380 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff3540b2200 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 10 client.25583 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559f8e318b60 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559f8e4ab380 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff354099b70 con 0x559f8e5a3de0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 10 client.25583 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 10 client.25583 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 20 client.25583 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e5b37d0 0x559f8e5b5bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 4 client.25583 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559f8e4e3700 con 0x559f8e5b37d0 2024-04-06T14:08:16.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.029+0000 7ff35bfff700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e5b37d0 0x559f8e5b5bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.030+0000 7ff35bfff700 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e5b37d0 0x559f8e5b5bc0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=47ee428f36efbab7 server_cookie=c31a59aae1838366 in_seq=0 out_seq=0 2024-04-06T14:08:16.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.030+0000 7ff33b7fe700 10 client.25583 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:16.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.030+0000 7ff33b7fe700 1 -- 172.21.15.90:0/4148396596 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x559f8e4e3700 con 0x559f8e5b37d0 2024-04-06T14:08:16.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.030+0000 7ff33b7fe700 10 client.25583 handle_command_reply: tid=0 2024-04-06T14:08:16.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.030+0000 7ff36ea28b80 1 client.25583 shutdown 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 msgr2=0x559f8e5a16a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7ff354099610 tx=0x7ff35401b460 comp rx=0 tx=0).stop 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 shutdown_connections 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e5b37d0 0x559f8e5b5bc0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e5a3de0 0x559f8e5a16a0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e599780 0x559f8e5a0c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/4148396596 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e59a0a0 0x559f8e5a1160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 >> 172.21.15.90:0/4148396596 conn(0x559f8e4f86a0 msgr2=0x559f8e593c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.032+0000 7ff33b7fe700 0 client.25583 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:16.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.033+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 shutdown_connections 2024-04-06T14:08:16.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.033+0000 7ff36ea28b80 1 -- 172.21.15.90:0/4148396596 wait complete. 2024-04-06T14:08:16.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.033+0000 7ff36ea28b80 20 client.25583 trim_cache size 0 max 16384 2024-04-06T14:08:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:15 smithi090 ceph-mon[43420]: pgmap v89: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 3.66k op/s 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.054+0000 7ff36ea28b80 1 Processor -- start 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 -- start start 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 0x559f8e5943a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e59a0a0 0x559f8e5948e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 0x559f8e59de80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e5a20b0 con 0x559f8e599780 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e5b35d0 con 0x559f8e59a0a0 2024-04-06T14:08:16.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e59f1a0 con 0x559f8e5a3de0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff35bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 0x559f8e5943a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff35bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 0x559f8e5943a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34928/0 (socket says 172.21.15.90:34928) 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff35bfff700 1 -- 172.21.15.90:0/2676930978 learned_addr learned my addr 172.21.15.90:0/2676930978 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e59a0a0 0x559f8e5948e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 0x559f8e59de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 -- 172.21.15.90:0/2676930978 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e59a0a0 msgr2=0x559f8e5948e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e59a0a0 0x559f8e5948e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 -- 172.21.15.90:0/2676930978 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 msgr2=0x559f8e5943a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 0x559f8e5943a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.055+0000 7ff360e11700 1 -- 172.21.15.90:0/2676930978 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559f8e4b0ea0 con 0x559f8e5a3de0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.056+0000 7ff360e11700 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 0x559f8e59de80 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7ff3540090a0 tx=0x7ff354004d20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fcb1298db37d8262 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.056+0000 7ff342ffd700 1 -- 172.21.15.90:0/2676930978 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff354098e50 con 0x559f8e5a3de0 2024-04-06T14:08:16.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.056+0000 7ff342ffd700 1 -- 172.21.15.90:0/2676930978 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff3540b2200 con 0x559f8e5a3de0 2024-04-06T14:08:16.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.056+0000 7ff342ffd700 1 -- 172.21.15.90:0/2676930978 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3540b90b0 con 0x559f8e5a3de0 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- 172.21.15.90:0/2676930978 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 msgr2=0x559f8e59de80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 0x559f8e59de80 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7ff3540090a0 tx=0x7ff354004d20 comp rx=0 tx=0).stop 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- 172.21.15.90:0/2676930978 shutdown_connections 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e5a3de0 0x559f8e59de80 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e599780 0x559f8e5943a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/2676930978 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e59a0a0 0x559f8e5948e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- 172.21.15.90:0/2676930978 >> 172.21.15.90:0/2676930978 conn(0x559f8e4f86a0 msgr2=0x559f8e59bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- 172.21.15.90:0/2676930978 shutdown_connections 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- 172.21.15.90:0/2676930978 wait complete. 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 -- start start 2024-04-06T14:08:16.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.057+0000 7ff36ea28b80 1 Processor -- start 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e599780 0x559f8e5aa910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e59a0a0 0x559f8e5a7da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e5a89c0 con 0x559f8e5a3de0 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e5a8b30 con 0x559f8e599780 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559f8e606590 con 0x559f8e59a0a0 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff36ea28b80 10 client.0 _ll_register_callbacks cb 0x559f8e490470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34944/0 (socket says 172.21.15.90:34944) 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 -- 172.21.15.90:0/3370480280 learned_addr learned my addr 172.21.15.90:0/3370480280 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff35bfff700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e599780 0x559f8e5aa910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff35b7fe700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e59a0a0 0x559f8e5a7da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 -- 172.21.15.90:0/3370480280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e59a0a0 msgr2=0x559f8e5a7da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e59a0a0 0x559f8e5a7da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 -- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e599780 msgr2=0x559f8e5aa910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e599780 0x559f8e5aa910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.058+0000 7ff360e11700 1 -- 172.21.15.90:0/3370480280 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559f8e318b60 con 0x559f8e5a3de0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff360e11700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7ff3540995c0 tx=0x7ff354097ec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9a995f862c39da81 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff33b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff3540b92d0 con 0x559f8e5a3de0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff3540a1d00 con 0x559f8e5a3de0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff34c01e040 con 0x559f8e5a3de0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff354099040 con 0x559f8e5a3de0 2024-04-06T14:08:16.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff36ea28b80 10 client.15930 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559f8e370cd0 con 0x559f8e5a3de0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.059+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff35407e020 con 0x559f8e5a3de0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff354084180 con 0x559f8e5a3de0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 10 client.15930 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 10 client.15930 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 20 client.15930 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e6077d0 0x559f8e609bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 4 client.15930 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x559f8e339cf0 con 0x559f8e6077d0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff35bfff700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e6077d0 0x559f8e609bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff33b7fe700 10 client.15930 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.060+0000 7ff35bfff700 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e6077d0 0x559f8e609bc0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e813c8873558cc36 server_cookie=d5782a23b25005dd in_seq=0 out_seq=0 2024-04-06T14:08:16.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff33b7fe700 1 -- 172.21.15.90:0/3370480280 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x559f8e339cf0 con 0x559f8e6077d0 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff33b7fe700 10 client.15930 handle_command_reply: tid=0 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 client.15930 shutdown 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 msgr2=0x559f8e5a82e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 secure :-1 s=READY pgs=510 cs=0 l=1 rev1=1 crypto rx=0x7ff3540995c0 tx=0x7ff354097ec0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 shutdown_connections 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559f8e6077d0 0x559f8e609bc0 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559f8e5a3de0 0x559f8e5a82e0 unknown :-1 s=CLOSED pgs=510 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff33b7fe700 0 client.15930 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559f8e599780 0x559f8e5aa910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 --2- 172.21.15.90:0/3370480280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559f8e59a0a0 0x559f8e5a7da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 >> 172.21.15.90:0/3370480280 conn(0x559f8e4f86a0 msgr2=0x559f8e59f4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 shutdown_connections 2024-04-06T14:08:16.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 1 -- 172.21.15.90:0/3370480280 wait complete. 2024-04-06T14:08:16.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff36ea28b80 20 client.15930 trim_cache size 0 max 16384 2024-04-06T14:08:16.067 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 msgr2=0x7ff3440a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 0x7ff3440a3750 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7ff35c1151c0 tx=0x7ff34c005060 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 msgr2=0x7ff35c113c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.061+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 0x7ff35c113c10 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7ff35400ef90 tx=0x7ff35400c3a0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 shutdown_connections 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff3440a1290 0x7ff3440a3750 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff35c11a450 0x7ff35c113c10 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff35c06c640 0x7ff35c119990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 --2- 172.21.15.90:0/2795229057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff35c06cf60 0x7ff35c119ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.062+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 >> 172.21.15.90:0/2795229057 conn(0x7ff35c06bd10 msgr2=0x7ff35c109dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.063+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 shutdown_connections 2024-04-06T14:08:16.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.063+0000 7ff362874700 1 -- 172.21.15.90:0/2795229057 wait complete. 2024-04-06T14:08:16.099 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:16.099 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:08:16.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:15 smithi134 ceph-mon[47549]: pgmap v89: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 3.66k op/s 2024-04-06T14:08:16.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.297+0000 7f9acd013700 1 -- 172.21.15.90:0/634209306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f4b0 msgr2=0x7f9ac8107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.297+0000 7f9acd013700 1 --2- 172.21.15.90:0/634209306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f4b0 0x7f9ac8107db0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f9ab00098f0 tx=0x7f9ab0009c00 comp rx=0 tx=0).stop 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 -- 172.21.15.90:0/634209306 shutdown_connections 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 --2- 172.21.15.90:0/634209306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 --2- 172.21.15.90:0/634209306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f4b0 0x7f9ac8107db0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 --2- 172.21.15.90:0/634209306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f920 0x7f9ac81082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 -- 172.21.15.90:0/634209306 >> 172.21.15.90:0/634209306 conn(0x7f9ac806be00 msgr2=0x7f9ac806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 -- 172.21.15.90:0/634209306 shutdown_connections 2024-04-06T14:08:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.298+0000 7f9acd013700 1 -- 172.21.15.90:0/634209306 wait complete. 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 Processor -- start 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 -- start start 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 0x7f9ac81a60d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f920 0x7f9ac81a6610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac81aa9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ac81aaff0 con 0x7f9ac810f4b0 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ac81ab160 con 0x7f9ac810f920 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9acd013700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ac81ab320 con 0x7f9ac811a590 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 0x7f9ac81a60d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 0x7f9ac81a60d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34958/0 (socket says 172.21.15.90:34958) 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2574340098 learned_addr learned my addr 172.21.15.90:0/2574340098 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.299+0000 7f9ac759e700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac81aa9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 -- 172.21.15.90:0/2574340098 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f920 msgr2=0x7f9ac81a6610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f920 0x7f9ac81a6610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 -- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 msgr2=0x7f9ac81a60d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 0x7f9ac81a60d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 -- 172.21.15.90:0/2574340098 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ab0009520 con 0x7f9ac811a590 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac759e700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac81aa9b0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f9ac000ef90 tx=0x7f9ac000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac4d99700 1 -- 172.21.15.90:0/2574340098 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ac0018070 con 0x7f9ac811a590 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ac81ab5a0 con 0x7f9ac811a590 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac4d99700 1 -- 172.21.15.90:0/2574340098 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ac000f040 con 0x7f9ac811a590 2024-04-06T14:08:16.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9ac4d99700 1 -- 172.21.15.90:0/2574340098 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ac000cc50 con 0x7f9ac811a590 2024-04-06T14:08:16.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.300+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ac81b2e40 con 0x7f9ac811a590 2024-04-06T14:08:16.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.305+0000 7f9ad91c7b80 1 Processor -- start 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 -- start start 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ac4d99700 1 -- 172.21.15.90:0/2574340098 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9ac0003a00 con 0x7f9ac811a590 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed68532b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 0x55ed6853c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed6853f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed6848a990 con 0x55ed6853cde0 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed68488350 con 0x55ed685330a0 2024-04-06T14:08:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.306+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed68483c40 con 0x55ed68532780 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9abffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed68532b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9abffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed68532b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:32942/0 (socket says 172.21.15.90:32942) 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9abffff700 1 -- 172.21.15.90:0/2113988523 learned_addr learned my addr 172.21.15.90:0/2113988523 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9abffff700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed6853f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 0x55ed6853c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2113988523 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 msgr2=0x55ed68532b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed68532b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2113988523 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 msgr2=0x55ed6853f1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed6853f1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2113988523 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ed683053b0 con 0x55ed685330a0 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.307+0000 7f9abffff700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed6853f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.308+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 0x55ed6853c8a0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f9ab0009810 tx=0x7f9ab0009400 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c4e524adac04bd1b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.309+0000 7f9ac4d99700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 0x7f9ab40a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.309+0000 7f9ac4d99700 1 -- 172.21.15.90:0/2574340098 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ac00cc880 con 0x7f9ac811a590 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.310+0000 7f9abd7fa700 1 -- 172.21.15.90:0/2113988523 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab000fd40 con 0x55ed685330a0 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.310+0000 7f9abd7fa700 1 -- 172.21.15.90:0/2113988523 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ab001d040 con 0x55ed685330a0 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.310+0000 7f9abd7fa700 1 -- 172.21.15.90:0/2113988523 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab0005040 con 0x55ed685330a0 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.310+0000 7f9abffff700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 0x7f9ab40a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2113988523 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 msgr2=0x55ed6853c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 0x55ed6853c8a0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f9ab0009810 tx=0x7f9ab0009400 comp rx=0 tx=0).stop 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9abffff700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 0x7f9ab40a3850 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x55ed685341f0 tx=0x7f9ab80058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2113988523 shutdown_connections 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed6853f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed68532b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2113988523 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed685330a0 0x55ed6853c8a0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.311+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2113988523 >> 172.21.15.90:0/2113988523 conn(0x55ed684916a0 msgr2=0x55ed6852da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.312+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2113988523 shutdown_connections 2024-04-06T14:08:16.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.312+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2113988523 wait complete. 2024-04-06T14:08:16.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.312+0000 7f9ad91c7b80 1 -- start start 2024-04-06T14:08:16.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.312+0000 7f9ad91c7b80 1 Processor -- start 2024-04-06T14:08:16.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed6853c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed6853cde0 0x55ed6853a190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed6841fd80 con 0x55ed685330a0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed6841fa20 con 0x55ed6853cde0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed683076a0 con 0x55ed68532780 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ad91c7b80 10 client.0 _ll_register_callbacks cb 0x55ed68429470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34980/0 (socket says 172.21.15.90:34980) 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 -- 172.21.15.90:0/1131308924 learned_addr learned my addr 172.21.15.90:0/1131308924 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed6853c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9ac759e700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed6853cde0 0x55ed6853a190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 -- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 msgr2=0x55ed6853c7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed6853c7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 -- 172.21.15.90:0/1131308924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed6853cde0 msgr2=0x55ed6853a190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed6853cde0 0x55ed6853a190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.313+0000 7f9abffff700 1 -- 172.21.15.90:0/1131308924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ac811d270 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9abffff700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7f9ab8010bb0 tx=0x7f9ab8009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ba1a1b66a7946cdc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab800fa60 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ab800fbc0 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ed68444380 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ab800fd80 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9ad91c7b80 10 client.15942 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ed682b1b60 con 0x55ed685330a0 2024-04-06T14:08:16.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ed68444380 con 0x55ed685330a0 2024-04-06T14:08:16.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.314+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ab8026020 con 0x55ed685330a0 2024-04-06T14:08:16.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 10 client.15942 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 10 client.15942 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:08:16.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 20 client.15942 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed6855d9a0 0x55ed6855fdc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 4 client.15942 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ed6847c700 con 0x55ed6855d9a0 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed6855d9a0 0x55ed6855fdc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.315+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed6855d9a0 0x55ed6855fdc0 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f9fb758eefdbfae4 server_cookie=26857c81714cecbf in_seq=0 out_seq=0 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9aae7fc700 10 client.15942 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9aae7fc700 1 -- 172.21.15.90:0/1131308924 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ed6847c700 con 0x55ed6855d9a0 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9aae7fc700 10 client.15942 handle_command_reply: tid=0 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9ad91c7b80 1 client.15942 shutdown 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 msgr2=0x55ed68539b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.317+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 secure :-1 s=READY pgs=511 cs=0 l=1 rev1=1 crypto rx=0x7f9ab8010bb0 tx=0x7f9ab8009260 comp rx=0 tx=0).stop 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 shutdown_connections 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed6855d9a0 0x55ed6855fdc0 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed6853cde0 0x55ed6853a190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed68532780 0x55ed6853c7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/1131308924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68539b10 unknown :-1 s=CLOSED pgs=511 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.319+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 >> 172.21.15.90:0/1131308924 conn(0x55ed684916a0 msgr2=0x55ed6852cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.320+0000 7f9aae7fc700 0 client.15942 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:16.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.331+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 shutdown_connections 2024-04-06T14:08:16.336 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.333+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/1131308924 wait complete. 2024-04-06T14:08:16.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.337+0000 7f9ad91c7b80 20 client.15942 trim_cache size 0 max 16384 2024-04-06T14:08:16.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.360+0000 7f9ad91c7b80 1 Processor -- start 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 -- start start 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68538420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed6853cde0 0x55ed68538960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed68527d20 con 0x55ed685330a0 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed6855b100 con 0x55ed68532780 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed6855a4f0 con 0x55ed6853cde0 2024-04-06T14:08:16.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38538/0 (socket says 172.21.15.90:38538) 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.361+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/998758436 learned_addr learned my addr 172.21.15.90:0/998758436 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.362+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/998758436 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed6853cde0 msgr2=0x55ed68538960 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.362+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed6853cde0 0x55ed68538960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.362+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/998758436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 msgr2=0x55ed68538420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.362+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68538420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.362+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/998758436 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ed68449ea0 con 0x55ed68532780 2024-04-06T14:08:16.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.364+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f9ab000b7e0 tx=0x7f9ab000b770 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4cbb5c660ea9a0e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.365+0000 7f9abdffb700 1 -- 172.21.15.90:0/998758436 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab0032330 con 0x55ed68532780 2024-04-06T14:08:16.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.365+0000 7f9abdffb700 1 -- 172.21.15.90:0/998758436 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ab0032490 con 0x55ed68532780 2024-04-06T14:08:16.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.365+0000 7f9abdffb700 1 -- 172.21.15.90:0/998758436 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab0032a70 con 0x55ed68532780 2024-04-06T14:08:16.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/998758436 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 msgr2=0x55ed68543a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f9ab000b7e0 tx=0x7f9ab000b770 comp rx=0 tx=0).stop 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/998758436 shutdown_connections 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed6853cde0 0x55ed68538960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68543a00 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/998758436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed685330a0 0x55ed68538420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/998758436 >> 172.21.15.90:0/998758436 conn(0x55ed684916a0 msgr2=0x55ed68534e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/998758436 shutdown_connections 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/998758436 wait complete. 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.367+0000 7f9ad91c7b80 1 -- start start 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 Processor -- start 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed685330a0 0x55ed68542110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed685b0790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed685b0cd0 con 0x55ed6853cde0 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed685b0e10 con 0x55ed68532780 2024-04-06T14:08:16.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ed685b0f50 con 0x55ed685330a0 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ad91c7b80 10 client.0 _ll_register_callbacks cb 0x55ed68429470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ac6d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38540/0 (socket says 172.21.15.90:38540) 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2781325353 learned_addr learned my addr 172.21.15.90:0/2781325353 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.368+0000 7f9abffff700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed685330a0 0x55ed68542110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed685330a0 msgr2=0x55ed68542110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed685330a0 0x55ed68542110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2781325353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 msgr2=0x55ed685b0790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed685b0790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 -- 172.21.15.90:0/2781325353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ed683053b0 con 0x55ed68532780 2024-04-06T14:08:16.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9ac6d9d700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f9ab0005990 tx=0x7f9ab000bfd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aed32258939df799 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9aae7fc700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:16.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ab000f840 con 0x55ed68532780 2024-04-06T14:08:16.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.369+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ab000f460 con 0x55ed68532780 2024-04-06T14:08:16.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.370+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9ab001d040 con 0x55ed68532780 2024-04-06T14:08:16.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.370+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ab000f9a0 con 0x55ed68532780 2024-04-06T14:08:16.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.370+0000 7f9ad91c7b80 10 client.25604 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.370+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ed68309cd0 con 0x55ed68532780 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.371+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ab001d040 con 0x55ed68532780 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ab0009c70 con 0x55ed68532780 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 10 client.25604 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 10 client.25604 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 20 client.25604 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed685b2220 0x55ed685b4610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 4 client.25604 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:16.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.373+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ed682d2cf0 con 0x55ed685b2220 2024-04-06T14:08:16.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.375+0000 7f9abffff700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed685b2220 0x55ed685b4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.376+0000 7f9abffff700 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed685b2220 0x55ed685b4610 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=47eb82d0ce05d223 server_cookie=25331cdc688b950c in_seq=0 out_seq=0 2024-04-06T14:08:16.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.376+0000 7f9aae7fc700 10 client.25604 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.377+0000 7f9aae7fc700 1 -- 172.21.15.90:0/2781325353 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ed682d2cf0 con 0x55ed685b2220 2024-04-06T14:08:16.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.377+0000 7f9aae7fc700 10 client.25604 handle_command_reply: tid=0 2024-04-06T14:08:16.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.377+0000 7f9ad91c7b80 1 client.25604 shutdown 2024-04-06T14:08:16.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.378+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 msgr2=0x55ed68541bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.378+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f9ab0005990 tx=0x7f9ab000bfd0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 shutdown_connections 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55ed685b2220 0x55ed685b4610 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ed6853cde0 0x55ed685b0790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ed68532780 0x55ed68541bd0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 --2- 172.21.15.90:0/2781325353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ed685330a0 0x55ed68542110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 >> 172.21.15.90:0/2781325353 conn(0x55ed684916a0 msgr2=0x55ed685383e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9aae7fc700 0 client.25604 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 shutdown_connections 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.379+0000 7f9ad91c7b80 1 -- 172.21.15.90:0/2781325353 wait complete. 2024-04-06T14:08:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.380+0000 7f9ad91c7b80 20 client.25604 trim_cache size 0 max 16384 2024-04-06T14:08:16.380 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.381+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 msgr2=0x7f9ab40a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.381+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 0x7f9ab40a3850 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x55ed685341f0 tx=0x7f9ab80058e0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.381+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 msgr2=0x7f9ac81aa9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.381+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac81aa9b0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f9ac000ef90 tx=0x7f9ac000c3a0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 shutdown_connections 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ab40a1390 0x7f9ab40a3850 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ac811a590 0x7f9ac81aa9b0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ac810f4b0 0x7f9ac81a60d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 --2- 172.21.15.90:0/2574340098 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ac810f920 0x7f9ac81a6610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.382+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 >> 172.21.15.90:0/2574340098 conn(0x7f9ac806be00 msgr2=0x7f9ac8118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.383+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 shutdown_connections 2024-04-06T14:08:16.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.383+0000 7f9acd013700 1 -- 172.21.15.90:0/2574340098 wait complete. 2024-04-06T14:08:16.404 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:16.405 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:08:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 -- 172.21.15.90:0/764356902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c100780 msgr2=0x7f596c100c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 --2- 172.21.15.90:0/764356902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c100780 0x7f596c100c00 secure :-1 s=READY pgs=512 cs=0 l=1 rev1=1 crypto rx=0x7f59600099c0 tx=0x7f5960009cd0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 -- 172.21.15.90:0/764356902 shutdown_connections 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 --2- 172.21.15.90:0/764356902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c101140 0x7f596c10caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 --2- 172.21.15.90:0/764356902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c104c10 0x7f596c104ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 --2- 172.21.15.90:0/764356902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c100780 0x7f596c100c00 unknown :-1 s=CLOSED pgs=512 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.559+0000 7f597224e700 1 -- 172.21.15.90:0/764356902 >> 172.21.15.90:0/764356902 conn(0x7f596c074ed0 msgr2=0x7f596c0752e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.560+0000 7f597224e700 1 -- 172.21.15.90:0/764356902 shutdown_connections 2024-04-06T14:08:16.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.563+0000 7f597224e700 1 -- 172.21.15.90:0/764356902 wait complete. 2024-04-06T14:08:16.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.564+0000 7f597224e700 1 Processor -- start 2024-04-06T14:08:16.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.564+0000 7f597224e700 1 -- start start 2024-04-06T14:08:16.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c101140 0x7f596c19e0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c104c10 0x7f596c1a2130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f596c1a2700 con 0x7f596c104c10 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f596c1a2870 con 0x7f596c100780 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f597224e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f596c1a2a30 con 0x7f596c101140 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c104c10 0x7f596c1a2130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38550/0 (socket says 172.21.15.90:38550) 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 -- 172.21.15.90:0/1399416782 learned_addr learned my addr 172.21.15.90:0/1399416782 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596affd700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c101140 0x7f596c19e0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 -- 172.21.15.90:0/1399416782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c101140 msgr2=0x7f596c19e0e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c101140 0x7f596c19e0e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 -- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c104c10 msgr2=0x7f596c1a2130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c104c10 0x7f596c1a2130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 -- 172.21.15.90:0/1399416782 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59600095f0 con 0x7f596c100780 2024-04-06T14:08:16.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.565+0000 7f596b7fe700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f595c00e970 tx=0x7f595c00ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.566+0000 7f5968ff9700 1 -- 172.21.15.90:0/1399416782 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f595c009790 con 0x7f596c100780 2024-04-06T14:08:16.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.566+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f596c1a2d10 con 0x7f596c100780 2024-04-06T14:08:16.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.566+0000 7f5968ff9700 1 -- 172.21.15.90:0/1399416782 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f595c00c7e0 con 0x7f596c100780 2024-04-06T14:08:16.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.566+0000 7f5968ff9700 1 -- 172.21.15.90:0/1399416782 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f595c003c70 con 0x7f596c100780 2024-04-06T14:08:16.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.566+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f596c06a9b0 con 0x7f596c100780 2024-04-06T14:08:16.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f5968ff9700 1 -- 172.21.15.90:0/1399416782 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f595c00c950 con 0x7f596c100780 2024-04-06T14:08:16.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 Processor -- start 2024-04-06T14:08:16.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 -- start start 2024-04-06T14:08:16.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922c780 0x55ad7922cb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922d0a0 0x55ad792368a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79184990 con 0x55ad7922d0a0 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79182350 con 0x55ad7922c780 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.571+0000 7f597e402b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad7917dc40 con 0x55ad79236de0 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f5968ff9700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 0x7f59540a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:32992/0 (socket says 172.21.15.90:32992) 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 -- 172.21.15.90:0/2259209334 learned_addr learned my addr 172.21.15.90:0/2259209334 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f5968ff9700 1 -- 172.21.15.90:0/1399416782 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f595c014070 con 0x7f596c100780 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 -- 172.21.15.90:0/2259209334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922c780 msgr2=0x55ad7922cb60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:16.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922c780 0x55ad7922cb60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 -- 172.21.15.90:0/2259209334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922d0a0 msgr2=0x55ad792368a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922d0a0 0x55ad792368a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 -- 172.21.15.90:0/2259209334 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad78fff3b0 con 0x55ad79236de0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.572+0000 7f596affd700 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f596000b580 tx=0x7f5960005fb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e4d32a1a3399365e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.573+0000 7f596bfff700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 0x7f59540a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.573+0000 7f5951ffb700 1 -- 172.21.15.90:0/2259209334 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f596001c050 con 0x55ad79236de0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.573+0000 7f5951ffb700 1 -- 172.21.15.90:0/2259209334 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f596001d040 con 0x55ad79236de0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.573+0000 7f5951ffb700 1 -- 172.21.15.90:0/2259209334 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f59600038c0 con 0x55ad79236de0 2024-04-06T14:08:16.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.573+0000 7f596bfff700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 0x7f59540a3750 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f5964003e20 tx=0x7f5964009130 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 -- 172.21.15.90:0/2259209334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 msgr2=0x55ad792391d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f596000b580 tx=0x7f5960005fb0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 -- 172.21.15.90:0/2259209334 shutdown_connections 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792391d0 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922c780 0x55ad7922cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 --2- 172.21.15.90:0/2259209334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922d0a0 0x55ad792368a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 -- 172.21.15.90:0/2259209334 >> 172.21.15.90:0/2259209334 conn(0x55ad7918b6a0 msgr2=0x55ad79227a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 -- 172.21.15.90:0/2259209334 shutdown_connections 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.574+0000 7f597e402b80 1 -- 172.21.15.90:0/2259209334 wait complete. 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.575+0000 7f597e402b80 1 -- start start 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.575+0000 7f597e402b80 1 Processor -- start 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad79236760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922d0a0 0x55ad79233af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79119d80 con 0x55ad7922c780 2024-04-06T14:08:16.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79119a20 con 0x55ad79236de0 2024-04-06T14:08:16.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad790016a0 con 0x55ad7922d0a0 2024-04-06T14:08:16.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f597e402b80 10 client.0 _ll_register_callbacks cb 0x55ad79123470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.576+0000 7f596bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38574/0 (socket says 172.21.15.90:38574) 2024-04-06T14:08:16.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 -- 172.21.15.90:0/835406802 learned_addr learned my addr 172.21.15.90:0/835406802 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596b7fe700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad79236760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 -- 172.21.15.90:0/835406802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922d0a0 msgr2=0x55ad79233af0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922d0a0 0x55ad79233af0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 -- 172.21.15.90:0/835406802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 msgr2=0x55ad79236760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad79236760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 -- 172.21.15.90:0/835406802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f596c10d3f0 con 0x55ad79236de0 2024-04-06T14:08:16.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f596bfff700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f59640087c0 tx=0x7f5964004b40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=52a78cd8218299d1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f594affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.577+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5964015850 con 0x55ad79236de0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.578+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f59640159b0 con 0x55ad79236de0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.578+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ad7913e380 con 0x55ad79236de0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5964015b70 con 0x55ad79236de0 2024-04-06T14:08:16.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 10 client.25613 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad78fabb60 con 0x55ad79236de0 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ad7913e380 con 0x55ad79236de0 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f59640163f0 con 0x55ad79236de0 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 10 client.25613 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 10 client.25613 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 20 client.25613 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad79257ad0 0x55ad79259ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 4 client.25613 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.579+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ad79176700 con 0x55ad79257ad0 2024-04-06T14:08:16.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.580+0000 7f596b7fe700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad79257ad0 0x55ad79259ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.580+0000 7f594affd700 10 client.25613 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.580+0000 7f596b7fe700 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad79257ad0 0x55ad79259ef0 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c55d435dc68c164 server_cookie=5438f0dbfbb79095 in_seq=0 out_seq=0 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f594affd700 1 -- 172.21.15.90:0/835406802 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ad79176700 con 0x55ad79257ad0 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f594affd700 10 client.25613 handle_command_reply: tid=0 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 client.25613 shutdown 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 msgr2=0x55ad79234120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f59640087c0 tx=0x7f5964004b40 comp rx=0 tx=0).stop 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 shutdown_connections 2024-04-06T14:08:16.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad79236de0 0x55ad79234120 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad79236760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad79257ad0 0x55ad79259ef0 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 --2- 172.21.15.90:0/835406802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922d0a0 0x55ad79233af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 >> 172.21.15.90:0/835406802 conn(0x55ad7918b6a0 msgr2=0x55ad79226c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f594affd700 0 client.25613 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 shutdown_connections 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.581+0000 7f597e402b80 1 -- 172.21.15.90:0/835406802 wait complete. 2024-04-06T14:08:16.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.582+0000 7f597e402b80 20 client.25613 trim_cache size 0 max 16384 2024-04-06T14:08:16.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.606+0000 7f597e402b80 1 Processor -- start 2024-04-06T14:08:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 -- start start 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad792273a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792278e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79234790 con 0x55ad7922c780 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad79228680 con 0x55ad7922d0a0 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.612+0000 7f597e402b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad7924f010 con 0x55ad79236de0 2024-04-06T14:08:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.615+0000 7f596bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792278e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35028/0 (socket says 172.21.15.90:35028) 2024-04-06T14:08:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 -- 172.21.15.90:0/2648017233 learned_addr learned my addr 172.21.15.90:0/2648017233 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 -- 172.21.15.90:0/2648017233 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 msgr2=0x55ad792278e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792278e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 -- 172.21.15.90:0/2648017233 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 msgr2=0x55ad792273a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad792273a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.617+0000 7f596b7fe700 1 -- 172.21.15.90:0/2648017233 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad79143ea0 con 0x55ad7922c780 2024-04-06T14:08:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.618+0000 7f596b7fe700 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f595c099b40 tx=0x7f595c00c450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=32fa3f04ca9784b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.618+0000 7f59527fc700 1 -- 172.21.15.90:0/2648017233 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f595c0a22c0 con 0x55ad7922c780 2024-04-06T14:08:16.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.618+0000 7f59527fc700 1 -- 172.21.15.90:0/2648017233 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f595c097f80 con 0x55ad7922c780 2024-04-06T14:08:16.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.618+0000 7f59527fc700 1 -- 172.21.15.90:0/2648017233 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f595c00cc50 con 0x55ad7922c780 2024-04-06T14:08:16.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.619+0000 7f597e402b80 1 -- 172.21.15.90:0/2648017233 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 msgr2=0x55ad7924eaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.619+0000 7f597e402b80 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 secure :-1 s=READY pgs=513 cs=0 l=1 rev1=1 crypto rx=0x7f595c099b40 tx=0x7f595c00c450 comp rx=0 tx=0).stop 2024-04-06T14:08:16.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.620+0000 7f597e402b80 1 -- 172.21.15.90:0/2648017233 shutdown_connections 2024-04-06T14:08:16.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.620+0000 7f597e402b80 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad79236de0 0x55ad792278e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.620+0000 7f597e402b80 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad7922c780 0x55ad7924eaa0 unknown :-1 s=CLOSED pgs=513 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.620+0000 7f597e402b80 1 --2- 172.21.15.90:0/2648017233 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad792273a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.620+0000 7f597e402b80 1 -- 172.21.15.90:0/2648017233 >> 172.21.15.90:0/2648017233 conn(0x55ad7918b6a0 msgr2=0x55ad792323e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.621+0000 7f597e402b80 1 -- 172.21.15.90:0/2648017233 shutdown_connections 2024-04-06T14:08:16.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.621+0000 7f597e402b80 1 -- 172.21.15.90:0/2648017233 wait complete. 2024-04-06T14:08:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.621+0000 7f597e402b80 1 -- start start 2024-04-06T14:08:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 Processor -- start 2024-04-06T14:08:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922c780 0x55ad7924e9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad79236de0 0x55ad7924d450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad7924db60 con 0x55ad79236de0 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad7924dcd0 con 0x55ad7922d0a0 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad792ab910 con 0x55ad7922c780 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f597e402b80 10 client.0 _ll_register_callbacks cb 0x55ad79123470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f596affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f596affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38590/0 (socket says 172.21.15.90:38590) 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.622+0000 7f596affd700 1 -- 172.21.15.90:0/3069355908 learned_addr learned my addr 172.21.15.90:0/3069355908 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596b7fe700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922c780 0x55ad7924e9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 -- 172.21.15.90:0/3069355908 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922c780 msgr2=0x55ad7924e9c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922c780 0x55ad7924e9c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 -- 172.21.15.90:0/3069355908 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad79236de0 msgr2=0x55ad7924d450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad79236de0 0x55ad7924d450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 -- 172.21.15.90:0/3069355908 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f596c06a9b0 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f596affd700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f596000b580 tx=0x7f596000be30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=77af2dd2bd17fb01 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f594affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5960003680 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f59600037e0 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.623+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f596001d040 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f596001c030 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 10 client.25616 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad79003cd0 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f596001d040 con 0x55ad7922d0a0 2024-04-06T14:08:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f596000f9f0 con 0x55ad7922d0a0 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 10 client.25616 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 10 client.25616 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 20 client.25616 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad792acc00 0x55ad792af020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 4 client.25616 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ad78fcccf0 con 0x55ad792acc00 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.624+0000 7f596b7fe700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad792acc00 0x55ad792af020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.625+0000 7f594affd700 10 client.25616 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.625+0000 7f596b7fe700 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad792acc00 0x55ad792af020 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=98db5a35ee941df2 server_cookie=11f2b2d061a8b5d5 in_seq=0 out_seq=0 2024-04-06T14:08:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.625+0000 7f594affd700 1 -- 172.21.15.90:0/3069355908 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ad78fcccf0 con 0x55ad792acc00 2024-04-06T14:08:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.625+0000 7f594affd700 10 client.25616 handle_command_reply: tid=0 2024-04-06T14:08:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.625+0000 7f597e402b80 1 client.25616 shutdown 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 msgr2=0x55ad7924ce90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f596000b580 tx=0x7f596000be30 comp rx=0 tx=0).stop 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 shutdown_connections 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad79236de0 0x55ad7924d450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad792acc00 0x55ad792af020 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad7922c780 0x55ad7924e9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 --2- 172.21.15.90:0/3069355908 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad7922d0a0 0x55ad7924ce90 secure :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f596000b580 tx=0x7f596000be30 comp rx=0 tx=0).stop 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 >> 172.21.15.90:0/3069355908 conn(0x55ad7918b6a0 msgr2=0x55ad792352f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.626+0000 7f594affd700 0 client.25616 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.627+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 shutdown_connections 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.628+0000 7f597e402b80 1 -- 172.21.15.90:0/3069355908 wait complete. 2024-04-06T14:08:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.628+0000 7f597e402b80 20 client.25616 trim_cache size 0 max 16384 2024-04-06T14:08:16.630 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 msgr2=0x7f59540a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 0x7f59540a3750 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f5964003e20 tx=0x7f5964009130 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 msgr2=0x7f596c19dba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f595c00e970 tx=0x7f595c00ec80 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 shutdown_connections 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f59540a1290 0x7f59540a3750 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f596c104c10 0x7f596c1a2130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f596c100780 0x7f596c19dba0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 --2- 172.21.15.90:0/1399416782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f596c101140 0x7f596c19e0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:16.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 >> 172.21.15.90:0/1399416782 conn(0x7f596c074ed0 msgr2=0x7f596c0feaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:16.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 shutdown_connections 2024-04-06T14:08:16.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:16.629+0000 7f597224e700 1 -- 172.21.15.90:0/1399416782 wait complete. 2024-04-06T14:08:16.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:16 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/29259475' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:17.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:16 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/29259475' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:17.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:16 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/29259475' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:17.645 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:08:17.646 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:08:17.646 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.865+0000 7fbb989e6700 1 -- 172.21.15.90:0/2295982930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9411a590 msgr2=0x7fbb9411c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.865+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2295982930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9411a590 0x7fbb9411c980 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c01c0d0 tx=0x7fbb8c01c3e0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.867+0000 7fbb989e6700 1 -- 172.21.15.90:0/2295982930 shutdown_connections 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.867+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2295982930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9411a590 0x7fbb9411c980 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.867+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2295982930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbb9410f4b0 0x7fbb94107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.867+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2295982930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9410f920 0x7fbb941082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.867+0000 7fbb989e6700 1 -- 172.21.15.90:0/2295982930 >> 172.21.15.90:0/2295982930 conn(0x7fbb9406be00 msgr2=0x7fbb9406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.874+0000 7fbb989e6700 1 -- 172.21.15.90:0/2295982930 shutdown_connections 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.877+0000 7fbb989e6700 1 -- 172.21.15.90:0/2295982930 wait complete. 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.878+0000 7fbb989e6700 1 Processor -- start 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.878+0000 7fbb989e6700 1 -- start start 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9410f4b0 0x7fbb941a2490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbb9410f920 0x7fbb941a29d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb94116220 con 0x7fbb9411a590 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb94116390 con 0x7fbb9410f4b0 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb989e6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb94116550 con 0x7fbb9410f920 2024-04-06T14:08:17.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35058/0 (socket says 172.21.15.90:35058) 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.879+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2813483729 learned_addr learned my addr 172.21.15.90:0/2813483729 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2813483729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbb9410f920 msgr2=0x7fbb941a29d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbb9410f920 0x7fbb941a29d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2813483729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9410f4b0 msgr2=0x7fbb941a2490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9410f4b0 0x7fbb941a2490 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2813483729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb8c01ad10 con 0x7fbb9411a590 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c0062a0 tx=0x7fbb8c003d50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb837fe700 1 -- 172.21.15.90:0/2813483729 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c01f070 con 0x7fbb9411a590 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb837fe700 1 -- 172.21.15.90:0/2813483729 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb8c0189e0 con 0x7fbb9411a590 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.880+0000 7fbb837fe700 1 -- 172.21.15.90:0/2813483729 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c02b750 con 0x7fbb9411a590 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.881+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb941167d0 con 0x7fbb9411a590 2024-04-06T14:08:17.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.881+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb941aa620 con 0x7fbb9411a590 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.885+0000 7fbb837fe700 1 -- 172.21.15.90:0/2813483729 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fbb8c0056e0 con 0x7fbb9411a590 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.886+0000 7fbb837fe700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 0x7fbb7c0a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.887+0000 7fbb837fe700 1 -- 172.21.15.90:0/2813483729 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbb8c0de4c0 con 0x7fbb9411a590 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.889+0000 7fbba4b9ab80 1 Processor -- start 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.889+0000 7fbb9259c700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 0x7fbb7c0a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.889+0000 7fbba4b9ab80 1 -- start start 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.889+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53ce4be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce5120 0x564f53cee920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53c3ca10 con 0x564f53ce5120 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53c3a3d0 con 0x564f53ceee60 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53c35cc0 con 0x564f53ce4800 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38610/0 (socket says 172.21.15.90:38610) 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.890+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2168017877 learned_addr learned my addr 172.21.15.90:0/2168017877 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2168017877 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 msgr2=0x564f53ce4be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53ce4be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2168017877 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce5120 msgr2=0x564f53cee920 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce5120 0x564f53cee920 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb92d9d700 1 -- 172.21.15.90:0/2168017877 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564f53ab7430 con 0x564f53ceee60 2024-04-06T14:08:17.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.892+0000 7fbb9259c700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 0x7fbb7c0a38e0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800ba10 tx=0x7fbb8800b3f0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.895+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c01e010 tx=0x7fbb8c01c820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b848c6a302b19e56 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.895+0000 7fbb80ff9700 1 -- 172.21.15.90:0/2168017877 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c0abe40 con 0x564f53ceee60 2024-04-06T14:08:17.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.895+0000 7fbb80ff9700 1 -- 172.21.15.90:0/2168017877 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb8c0ab740 con 0x564f53ceee60 2024-04-06T14:08:17.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.895+0000 7fbb80ff9700 1 -- 172.21.15.90:0/2168017877 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c0af370 con 0x564f53ceee60 2024-04-06T14:08:17.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.896+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2168017877 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 msgr2=0x564f53cf1250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.896+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c01e010 tx=0x7fbb8c01c820 comp rx=0 tx=0).stop 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.897+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2168017877 shutdown_connections 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.897+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cf1250 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.897+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53ce4be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.897+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2168017877 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce5120 0x564f53cee920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.897+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2168017877 >> 172.21.15.90:0/2168017877 conn(0x564f53c43720 msgr2=0x564f53cdfb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.898+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2168017877 shutdown_connections 2024-04-06T14:08:17.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.898+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2168017877 wait complete. 2024-04-06T14:08:17.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.898+0000 7fbba4b9ab80 1 -- start start 2024-04-06T14:08:17.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.898+0000 7fbba4b9ab80 1 Processor -- start 2024-04-06T14:08:17.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 0x564f53cebd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce5120 0x564f53cec2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53bd1e00 con 0x564f53ce4800 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53bd1aa0 con 0x564f53ceee60 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53ab9720 con 0x564f53ce5120 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbba4b9ab80 10 client.0 _ll_register_callbacks cb 0x564f53bdb4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38614/0 (socket says 172.21.15.90:38614) 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 -- 172.21.15.90:0/4121286760 learned_addr learned my addr 172.21.15.90:0/4121286760 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb9259c700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 0x564f53cebd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 -- 172.21.15.90:0/4121286760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce5120 msgr2=0x564f53cec2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce5120 0x564f53cec2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 -- 172.21.15.90:0/4121286760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 msgr2=0x564f53cebd70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 0x564f53cebd70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 -- 172.21.15.90:0/4121286760 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb941aa620 con 0x564f53ceee60 2024-04-06T14:08:17.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb9259c700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 0x564f53cebd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:08:17.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c0ab6d0 tx=0x7fbb8c005a80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8649324dc5e88830 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.899+0000 7fbb79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8c0cb070 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbb8c0ab9a0 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564f53bf6400 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbb8c0af380 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbba4b9ab80 10 client.25625 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564f53a63be0 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.900+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564f53bf6400 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbb8c0b3730 con 0x564f53ceee60 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 10 client.25625 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:17.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 10 client.25625 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 20 client.25625 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53cfe390 0x564f53d007b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 4 client.25625 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.901+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564f53c2e780 con 0x564f53cfe390 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.903+0000 7fbb9259c700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53cfe390 0x564f53d007b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.903+0000 7fbb79ffb700 10 client.25625 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:17.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.903+0000 7fbb9259c700 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53cfe390 0x564f53d007b0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=830d88ab4edb5466 server_cookie=c7d4c677a513e282 in_seq=0 out_seq=0 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbb79ffb700 1 -- 172.21.15.90:0/4121286760 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564f53c2e780 con 0x564f53cfe390 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbb79ffb700 10 client.25625 handle_command_reply: tid=0 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 client.25625 shutdown 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 msgr2=0x564f53cec7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c0ab6d0 tx=0x7fbb8c005a80 comp rx=0 tx=0).stop 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 shutdown_connections 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ceee60 0x564f53cec7f0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53ce4800 0x564f53cebd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53cfe390 0x564f53d007b0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/4121286760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce5120 0x564f53cec2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 >> 172.21.15.90:0/4121286760 conn(0x564f53c43720 msgr2=0x564f53cdeca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbb79ffb700 0 client.25625 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:17.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 shutdown_connections 2024-04-06T14:08:17.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.904+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/4121286760 wait complete. 2024-04-06T14:08:17.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.905+0000 7fbba4b9ab80 20 client.25625 trim_cache size 0 max 16384 2024-04-06T14:08:17.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 Processor -- start 2024-04-06T14:08:17.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 -- start start 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53ceaed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53ce3f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53cd9da0 con 0x564f53c43720 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53ced2d0 con 0x564f53ce5120 2024-04-06T14:08:17.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53ce5500 con 0x564f53ce4800 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbb91d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbb91d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33044/0 (socket says 172.21.15.90:33044) 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.933+0000 7fbb91d9b700 1 -- 172.21.15.90:0/2368850558 learned_addr learned my addr 172.21.15.90:0/2368850558 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53ce3f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 -- 172.21.15.90:0/2368850558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 msgr2=0x564f53ce3f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53ce3f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 -- 172.21.15.90:0/2368850558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 msgr2=0x564f53ceaed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53ceaed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 -- 172.21.15.90:0/2368850558 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564f53cdbbe0 con 0x564f53ce4800 2024-04-06T14:08:17.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fbb84009590 tx=0x7fbb840098a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ec2d5e5dfc6b2591 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb817fa700 1 -- 172.21.15.90:0/2368850558 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb840107b0 con 0x564f53ce4800 2024-04-06T14:08:17.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb817fa700 1 -- 172.21.15.90:0/2368850558 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb84010dd0 con 0x564f53ce4800 2024-04-06T14:08:17.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.934+0000 7fbb817fa700 1 -- 172.21.15.90:0/2368850558 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb84011ea0 con 0x564f53ce4800 2024-04-06T14:08:17.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2368850558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 msgr2=0x564f53cf5850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fbb84009590 tx=0x7fbb840098a0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2368850558 shutdown_connections 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53ce3f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53ceaed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/2368850558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf5850 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.935+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2368850558 >> 172.21.15.90:0/2368850558 conn(0x564f53ce6980 msgr2=0x564f53ce37f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.936+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2368850558 shutdown_connections 2024-04-06T14:08:17.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/2368850558 wait complete. 2024-04-06T14:08:17.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 -- start start 2024-04-06T14:08:17.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 Processor -- start 2024-04-06T14:08:17.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf4860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53cf4da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53d52350 con 0x564f53c43720 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53d524c0 con 0x564f53ce5120 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564f53d52680 con 0x564f53ce4800 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbba4b9ab80 10 client.0 _ll_register_callbacks cb 0x564f53bdb4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.937+0000 7fbb9259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35082/0 (socket says 172.21.15.90:35082) 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 -- 172.21.15.90:0/1809594879 learned_addr learned my addr 172.21.15.90:0/1809594879 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf4860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb92d9d700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53cf4da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 -- 172.21.15.90:0/1809594879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 msgr2=0x564f53cf4860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf4860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 -- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 msgr2=0x564f53cf4da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:17.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53cf4da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 -- 172.21.15.90:0/1809594879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb84009270 con 0x564f53c43720 2024-04-06T14:08:17.940 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb9259c700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800d980 tx=0x7fbb8800de50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fc05bdbfc9c7367 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.938+0000 7fbb79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb88022560 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fbb88017040 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbb8c0ab740 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbb88010460 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbba4b9ab80 10 client.15960 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564f53bfbf20 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.939+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb8c0ab740 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbb8801e560 con 0x564f53c43720 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 10 client.15960 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 10 client.15960 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 20 client.15960 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53ceee60 0x564f53d55f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:17.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 4 client.15960 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:08:17.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564f53a84d70 con 0x564f53ceee60 2024-04-06T14:08:17.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53ceee60 0x564f53d55f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:17.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbb79ffb700 10 client.15960 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:17.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.940+0000 7fbb91d9b700 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53ceee60 0x564f53d55f50 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b1de0b2b89e940b6 server_cookie=82b5ba6f46f102b4 in_seq=0 out_seq=0 2024-04-06T14:08:17.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.944+0000 7fbb79ffb700 1 -- 172.21.15.90:0/1809594879 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564f53a84d70 con 0x564f53ceee60 2024-04-06T14:08:17.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.944+0000 7fbb79ffb700 10 client.15960 handle_command_reply: tid=0 2024-04-06T14:08:17.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.944+0000 7fbba4b9ab80 1 client.15960 shutdown 2024-04-06T14:08:17.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.944+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 msgr2=0x564f53cf4320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.944+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 secure :-1 s=READY pgs=515 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800d980 tx=0x7fbb8800de50 comp rx=0 tx=0).stop 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 shutdown_connections 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564f53ce5120 0x564f53cf4da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564f53c43720 0x564f53cf4320 unknown :-1 s=CLOSED pgs=515 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564f53ceee60 0x564f53d55f50 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 --2- 172.21.15.90:0/1809594879 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564f53ce4800 0x564f53cf4860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 >> 172.21.15.90:0/1809594879 conn(0x564f53ce6980 msgr2=0x564f53ce4080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.947+0000 7fbb79ffb700 0 client.15960 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:17.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.951+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 shutdown_connections 2024-04-06T14:08:17.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.951+0000 7fbba4b9ab80 1 -- 172.21.15.90:0/1809594879 wait complete. 2024-04-06T14:08:17.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.951+0000 7fbba4b9ab80 20 client.15960 trim_cache size 0 max 16384 2024-04-06T14:08:17.951 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:17.951 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:08:17.951 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "811b2bc6-faf7-4418-89bb-89cb7b941947", 2024-04-06T14:08:17.951 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:08:17.951 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:17.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.952+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 msgr2=0x7fbb7c0a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.952+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 0x7fbb7c0a38e0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800ba10 tx=0x7fbb8800b3f0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.952+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 msgr2=0x7fbb94115bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:17.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.952+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 secure :-1 s=READY pgs=514 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c0062a0 tx=0x7fbb8c003d50 comp rx=0 tx=0).stop 2024-04-06T14:08:17.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 shutdown_connections 2024-04-06T14:08:17.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbb7c0a1420 0x7fbb7c0a38e0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbb9411a590 0x7fbb94115bc0 unknown :-1 s=CLOSED pgs=514 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbb9410f4b0 0x7fbb941a2490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 --2- 172.21.15.90:0/2813483729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbb9410f920 0x7fbb941a29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:17.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.955+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 >> 172.21.15.90:0/2813483729 conn(0x7fbb9406be00 msgr2=0x7fbb9411a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:17.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.961+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 shutdown_connections 2024-04-06T14:08:17.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:17.961+0000 7fbb989e6700 1 -- 172.21.15.90:0/2813483729 wait complete. 2024-04-06T14:08:17.988 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:08:17.988 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:08:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 150.0 s, 2112.1 tps, lat 48.348 ms stddev 70.062 2024-04-06T14:08:18.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2493919657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8104700 msgr2=0x7f9dd810c280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2493919657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8104700 0x7f9dd810c280 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f9dcc0098f0 tx=0x7f9dcc009c00 comp rx=0 tx=0).stop 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2493919657 shutdown_connections 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2493919657 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8104700 0x7f9dd810c280 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2493919657 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd80ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2493919657 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80fff90 0x7f9dd8103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.199+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2493919657 >> 172.21.15.90:0/2493919657 conn(0x7f9dd80761b0 msgr2=0x7f9dd80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.200+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2493919657 shutdown_connections 2024-04-06T14:08:18.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.200+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2493919657 wait complete. 2024-04-06T14:08:18.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 Processor -- start 2024-04-06T14:08:18.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 -- start start 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd8194fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 0x7f9dd81954e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8199880 0x7f9dd8199d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dd819a300 con 0x7f9dd8104700 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dd819a470 con 0x7f9dd8199880 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.201+0000 7f9ddd6da700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dd819a630 con 0x7f9dd80ff5e0 2024-04-06T14:08:18.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.202+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd8194fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.202+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd8194fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33068/0 (socket says 172.21.15.90:33068) 2024-04-06T14:08:18.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.202+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/2983913334 learned_addr learned my addr 172.21.15.90:0/2983913334 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:18.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.202+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 0x7f9dd81954e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2983913334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 msgr2=0x7f9dd8194fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd8194fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2983913334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8199880 msgr2=0x7f9dd8199d00 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:08:18.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8199880 0x7f9dd8199d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2983913334 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9dcc009520 con 0x7f9dd8104700 2024-04-06T14:08:18.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 0x7f9dd81954e0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f9dd0007e90 tx=0x7f9dd000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.203+0000 7f9dbffff700 1 -- 172.21.15.90:0/2983913334 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd000ddd0 con 0x7f9dd8104700 2024-04-06T14:08:18.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.204+0000 7f9dbffff700 1 -- 172.21.15.90:0/2983913334 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dd000f040 con 0x7f9dd8104700 2024-04-06T14:08:18.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.204+0000 7f9dbffff700 1 -- 172.21.15.90:0/2983913334 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd00059c0 con 0x7f9dd8104700 2024-04-06T14:08:18.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.204+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9dd81a1b90 con 0x7f9dd8104700 2024-04-06T14:08:18.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.204+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dd81a20a0 con 0x7f9dd8104700 2024-04-06T14:08:18.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.209+0000 7f9dbffff700 1 -- 172.21.15.90:0/2983913334 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9dd0004ad0 con 0x7f9dd8104700 2024-04-06T14:08:18.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9dbffff700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 0x7f9dc00a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 Processor -- start 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9dbffff700 1 -- 172.21.15.90:0/2983913334 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9dd00cc4c0 con 0x7f9dd8104700 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 -- start start 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5a60a0 0x55a93d5c08b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5c31e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d4fd990 con 0x55a93d5c0df0 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d4fb350 con 0x55a93d5a5780 2024-04-06T14:08:18.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9de988eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d4f6c40 con 0x55a93d5a60a0 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.211+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38650/0 (socket says 172.21.15.90:38650) 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3365980547 learned_addr learned my addr 172.21.15.90:0/3365980547 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 0x7f9dc00a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3365980547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5a60a0 msgr2=0x55a93d5c08b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5a60a0 0x55a93d5c08b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3365980547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 msgr2=0x55a93d5c31e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5c31e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3365980547 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a93d3783b0 con 0x55a93d5a5780 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8012850 tx=0x7f9dc8012b60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=65d6733facf87f8f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.212+0000 7f9dbd7fa700 1 -- 172.21.15.90:0/3365980547 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dc8004830 con 0x55a93d5a5780 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.213+0000 7f9dbd7fa700 1 -- 172.21.15.90:0/3365980547 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dc801dba0 con 0x55a93d5a5780 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.213+0000 7f9dbd7fa700 1 -- 172.21.15.90:0/3365980547 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dc8011980 con 0x55a93d5a5780 2024-04-06T14:08:18.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.213+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 0x7f9dc00a38e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8021000 tx=0x7f9dc801d000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 -- 172.21.15.90:0/3365980547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 msgr2=0x55a93d5a5b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8012850 tx=0x7f9dc8012b60 comp rx=0 tx=0).stop 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 -- 172.21.15.90:0/3365980547 shutdown_connections 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5c31e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a5780 0x55a93d5a5b60 secure :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8012850 tx=0x7f9dc8012b60 comp rx=0 tx=0).stop 2024-04-06T14:08:18.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3365980547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5a60a0 0x55a93d5c08b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 -- 172.21.15.90:0/3365980547 >> 172.21.15.90:0/3365980547 conn(0x55a93d5046a0 msgr2=0x55a93d5a0a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 -- 172.21.15.90:0/3365980547 shutdown_connections 2024-04-06T14:08:18.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.214+0000 7f9de988eb80 1 -- 172.21.15.90:0/3365980547 wait complete. 2024-04-06T14:08:18.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 -- start start 2024-04-06T14:08:18.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 Processor -- start 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5be210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5ccb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d492d80 con 0x55a93d5c0df0 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d492a20 con 0x55a93d5a60a0 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d37a6a0 con 0x55a93d5be750 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9de988eb80 10 client.0 _ll_register_callbacks cb 0x55a93d49c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9dd6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38662/0 (socket says 172.21.15.90:38662) 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.215+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3068565000 learned_addr learned my addr 172.21.15.90:0/3068565000 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3068565000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 msgr2=0x55a93d5ccb20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:18.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5ccb20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3068565000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 msgr2=0x55a93d5be210 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5be210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/3068565000 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dc80124b0 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8015000 tx=0x7f9dc8012f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dffffc833e39bec5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dbaff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dc80116a0 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.216+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9dc8011800 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a93d4b7380 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9dc8011ba0 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 10 client.25640 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a93d324b60 con 0x55a93d5a60a0 2024-04-06T14:08:18.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dc803f080 con 0x55a93d5a60a0 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9dc80264e0 con 0x55a93d5a60a0 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 10 client.25640 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 10 client.25640 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 20 client.25640 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d5d0ee0 0x55a93d5d3300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 4 client.25640 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.217+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a93d4ef700 con 0x55a93d5d0ee0 2024-04-06T14:08:18.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.218+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d5d0ee0 0x55a93d5d3300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.218+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d5d0ee0 0x55a93d5d3300 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=700a6090326b54cb server_cookie=7fed5533ed36087f in_seq=0 out_seq=0 2024-04-06T14:08:18.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.218+0000 7f9dbaff5700 10 client.25640 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:18.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9dbaff5700 1 -- 172.21.15.90:0/3068565000 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a93d4ef700 con 0x55a93d5d0ee0 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9dbaff5700 10 client.25640 handle_command_reply: tid=0 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 client.25640 shutdown 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 msgr2=0x55a93d5bdcd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8015000 tx=0x7f9dc8012f60 comp rx=0 tx=0).stop 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 shutdown_connections 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d5d0ee0 0x55a93d5d3300 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5ccb20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5a60a0 0x55a93d5bdcd0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9dbaff5700 0 client.25640 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 --2- 172.21.15.90:0/3068565000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5c0df0 0x55a93d5be210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 >> 172.21.15.90:0/3068565000 conn(0x55a93d5046a0 msgr2=0x55a93d59fc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.219+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 shutdown_connections 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.220+0000 7f9de988eb80 1 -- 172.21.15.90:0/3068565000 wait complete. 2024-04-06T14:08:18.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.220+0000 7f9de988eb80 20 client.25640 trim_cache size 0 max 16384 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 Processor -- start 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 -- start start 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d5bcc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5c0df0 0x55a93d5a08e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5c3870 con 0x55a93d5a60a0 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5a1dc0 con 0x55a93d5c0df0 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.264+0000 7f9de988eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5983c0 con 0x55a93d5be750 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33082/0 (socket says 172.21.15.90:33082) 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2730249159 learned_addr learned my addr 172.21.15.90:0/2730249159 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2730249159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5c0df0 msgr2=0x55a93d5a08e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5c0df0 0x55a93d5a08e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2730249159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 msgr2=0x55a93d5bcc20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d5bcc20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 -- 172.21.15.90:0/2730249159 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a93d4bcea0 con 0x55a93d5be750 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f9dd0096010 tx=0x7f9dd00b9040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c04f788b41ab4eee server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dbdffb700 1 -- 172.21.15.90:0/2730249159 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd00b1070 con 0x55a93d5be750 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dbdffb700 1 -- 172.21.15.90:0/2730249159 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dd0098f00 con 0x55a93d5be750 2024-04-06T14:08:18.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.265+0000 7f9dbdffb700 1 -- 172.21.15.90:0/2730249159 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dd00b78b0 con 0x55a93d5be750 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.266+0000 7f9de988eb80 1 -- 172.21.15.90:0/2730249159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 msgr2=0x55a93d5a03a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.266+0000 7f9de988eb80 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f9dd0096010 tx=0x7f9dd00b9040 comp rx=0 tx=0).stop 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 -- 172.21.15.90:0/2730249159 shutdown_connections 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5c0df0 0x55a93d5a08e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d5bcc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 --2- 172.21.15.90:0/2730249159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5be750 0x55a93d5a03a0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 -- 172.21.15.90:0/2730249159 >> 172.21.15.90:0/2730249159 conn(0x55a93d5046a0 msgr2=0x55a93d5bfdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 -- 172.21.15.90:0/2730249159 shutdown_connections 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.267+0000 7f9de988eb80 1 -- 172.21.15.90:0/2730249159 wait complete. 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 -- start start 2024-04-06T14:08:18.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 Processor -- start 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d61f020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 0x55a93d61f560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5c0df0 0x55a93d5c68f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5c6f40 con 0x55a93d5a60a0 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5c70b0 con 0x55a93d5be750 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a93d5c7270 con 0x55a93d5c0df0 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9de988eb80 10 client.0 _ll_register_callbacks cb 0x55a93d49c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9dd67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 0x55a93d61f560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9dd67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 0x55a93d61f560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38684/0 (socket says 172.21.15.90:38684) 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.268+0000 7f9dd67fc700 1 -- 172.21.15.90:0/920383634 learned_addr learned my addr 172.21.15.90:0/920383634 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:18.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d61f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/920383634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5c0df0 msgr2=0x55a93d5c68f0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5c0df0 0x55a93d5c68f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 msgr2=0x55a93d61f560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 0x55a93d61f560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 -- 172.21.15.90:0/920383634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dd81a20a0 con 0x55a93d5a60a0 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dd6ffd700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d61f020 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f9dc800b350 tx=0x7f9dc8005ce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=474885bdb767f0fe server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.269+0000 7f9dbaff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:18.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.270+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9dc8026a90 con 0x55a93d5a60a0 2024-04-06T14:08:18.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.270+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9dc8042070 con 0x55a93d5a60a0 2024-04-06T14:08:18.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.270+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9dc801dba0 con 0x55a93d5a60a0 2024-04-06T14:08:18.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.270+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9dc8026660 con 0x55a93d5a60a0 2024-04-06T14:08:18.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.272+0000 7f9de988eb80 10 client.15969 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:18.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.272+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a93d37ccd0 con 0x55a93d5a60a0 2024-04-06T14:08:18.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9dc801dba0 con 0x55a93d5a60a0 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9dc80303a0 con 0x55a93d5a60a0 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 10 client.15969 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 10 client.15969 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 20 client.15969 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d626b10 0x55a93d628f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 4 client.15969 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55a93d345cf0 con 0x55a93d626b10 2024-04-06T14:08:18.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.274+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d626b10 0x55a93d628f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:18.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.275+0000 7f9dbaff5700 10 client.15969 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:18.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.275+0000 7f9dd67fc700 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d626b10 0x55a93d628f30 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e83c7f4616a15c96 server_cookie=e00ceb24e0135e01 in_seq=0 out_seq=0 2024-04-06T14:08:18.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9dbaff5700 1 -- 172.21.15.90:0/920383634 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55a93d345cf0 con 0x55a93d626b10 2024-04-06T14:08:18.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9dbaff5700 10 client.15969 handle_command_reply: tid=0 2024-04-06T14:08:18.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 client.15969 shutdown 2024-04-06T14:08:18.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 msgr2=0x55a93d61f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d61f020 secure :-1 s=READY pgs=517 cs=0 l=1 rev1=1 crypto rx=0x7f9dc800b350 tx=0x7f9dc8005ce0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 shutdown_connections 2024-04-06T14:08:18.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a93d626b10 0x55a93d628f30 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a93d5c0df0 0x55a93d5c68f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a93d5a60a0 0x55a93d61f020 unknown :-1 s=CLOSED pgs=517 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 --2- 172.21.15.90:0/920383634 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a93d5be750 0x55a93d61f560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 >> 172.21.15.90:0/920383634 conn(0x55a93d5046a0 msgr2=0x55a93d5bf520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9dbaff5700 0 client.15969 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.277+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 shutdown_connections 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.278+0000 7f9de988eb80 1 -- 172.21.15.90:0/920383634 wait complete. 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.278+0000 7f9de988eb80 20 client.15969 trim_cache size 0 max 16384 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:18.278 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: "811b2bc6-faf7-4418-89bb-89cb7b941947": { 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "811b2bc6-faf7-4418-89bb-89cb7b941947", 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:08:18.279 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 msgr2=0x7f9dc00a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 0x7f9dc00a38e0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7f9dc8021000 tx=0x7f9dc801d000 comp rx=0 tx=0).stop 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 msgr2=0x7f9dd81954e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 0x7f9dd81954e0 secure :-1 s=READY pgs=516 cs=0 l=1 rev1=1 crypto rx=0x7f9dd0007e90 tx=0x7f9dd000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 shutdown_connections 2024-04-06T14:08:18.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9dc00a1420 0x7f9dc00a38e0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9dd8199880 0x7f9dd8199d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9dd80ff5e0 0x7f9dd8194fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 --2- 172.21.15.90:0/2983913334 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9dd8104700 0x7f9dd81954e0 unknown :-1 s=CLOSED pgs=516 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:18.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.279+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 >> 172.21.15.90:0/2983913334 conn(0x7f9dd80761b0 msgr2=0x7f9dd8102890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:18.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.280+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 shutdown_connections 2024-04-06T14:08:18.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:18.281+0000 7f9ddd6da700 1 -- 172.21.15.90:0/2983913334 wait complete. 2024-04-06T14:08:18.296 INFO:tasks.cephfs.filesystem:scrub status for tag:811b2bc6-faf7-4418-89bb-89cb7b941947 - {'path': '/', 'tag': '811b2bc6-faf7-4418-89bb-89cb7b941947', 'options': 'recursive,force'} 2024-04-06T14:08:18.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:18 smithi134 ceph-mon[47549]: pgmap v90: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 38 MiB/s wr, 2.13k op/s 2024-04-06T14:08:18.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:18 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:08:18.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:18 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:18.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:18 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:08:18.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:18 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:18 smithi105 ceph-mon[46969]: pgmap v90: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 38 MiB/s wr, 2.13k op/s 2024-04-06T14:08:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:18 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:08:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:18 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:18 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:08:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:18 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:18 smithi090 ceph-mon[43420]: pgmap v90: 97 pgs: 97 active+clean; 4.5 GiB data, 5.9 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 38 MiB/s wr, 2.13k op/s 2024-04-06T14:08:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:18 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:08:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:18 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:18 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:08:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:18 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:19.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:19 smithi105 ceph-mon[46969]: pgmap v91: 97 pgs: 97 active+clean; 4.5 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 44 MiB/s wr, 2.43k op/s 2024-04-06T14:08:20.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:19 smithi090 ceph-mon[43420]: pgmap v91: 97 pgs: 97 active+clean; 4.5 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 44 MiB/s wr, 2.43k op/s 2024-04-06T14:08:20.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:19 smithi134 ceph-mon[47549]: pgmap v91: 97 pgs: 97 active+clean; 4.5 GiB data, 6.0 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 44 MiB/s wr, 2.43k op/s 2024-04-06T14:08:20.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:20 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:08:21.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:20 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:08:21.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:20 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:08:21.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:21 smithi105 ceph-mon[46969]: pgmap v92: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 41 MiB/s wr, 2.32k op/s 2024-04-06T14:08:22.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:21 smithi090 ceph-mon[43420]: pgmap v92: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 41 MiB/s wr, 2.32k op/s 2024-04-06T14:08:22.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:21 smithi134 ceph-mon[47549]: pgmap v92: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 41 MiB/s wr, 2.32k op/s 2024-04-06T14:08:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 155.0 s, 3462.7 tps, lat 28.810 ms stddev 32.160 2024-04-06T14:08:23.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:23 smithi105 ceph-mon[46969]: pgmap v93: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 34 MiB/s wr, 2.07k op/s 2024-04-06T14:08:24.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:23 smithi090 ceph-mon[43420]: pgmap v93: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 34 MiB/s wr, 2.07k op/s 2024-04-06T14:08:24.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:23 smithi134 ceph-mon[47549]: pgmap v93: 97 pgs: 97 active+clean; 4.6 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 34 MiB/s wr, 2.07k op/s 2024-04-06T14:08:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:25 smithi105 ceph-mon[46969]: pgmap v94: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 44 MiB/s wr, 2.49k op/s 2024-04-06T14:08:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:25 smithi090 ceph-mon[43420]: pgmap v94: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 44 MiB/s wr, 2.49k op/s 2024-04-06T14:08:26.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:25 smithi134 ceph-mon[47549]: pgmap v94: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 44 MiB/s wr, 2.49k op/s 2024-04-06T14:08:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 160.0 s, 3347.6 tps, lat 29.989 ms stddev 32.902 2024-04-06T14:08:28.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:28 smithi134 ceph-mon[47549]: pgmap v95: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 21 MiB/s wr, 918 op/s 2024-04-06T14:08:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:28 smithi105 ceph-mon[46969]: pgmap v95: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 21 MiB/s wr, 918 op/s 2024-04-06T14:08:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:28 smithi090 ceph-mon[43420]: pgmap v95: 97 pgs: 97 active+clean; 4.6 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 21 MiB/s wr, 918 op/s 2024-04-06T14:08:29.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:29 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:08:29.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:29 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:08:30.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:29 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:08:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:30 smithi134 ceph-mon[47549]: pgmap v96: 97 pgs: 97 active+clean; 4.7 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 24 MiB/s wr, 1.06k op/s 2024-04-06T14:08:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:08:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:08:30.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:08:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:30 smithi105 ceph-mon[46969]: pgmap v96: 97 pgs: 97 active+clean; 4.7 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 24 MiB/s wr, 1.06k op/s 2024-04-06T14:08:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:08:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:08:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:08:31.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:30 smithi090 ceph-mon[43420]: pgmap v96: 97 pgs: 97 active+clean; 4.7 GiB data, 6.3 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 24 MiB/s wr, 1.06k op/s 2024-04-06T14:08:31.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:08:31.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:08:31.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:08:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:31 smithi105 ceph-mon[46969]: pgmap v97: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 23 MiB/s wr, 984 op/s 2024-04-06T14:08:32.037 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:31 smithi090 ceph-mon[43420]: pgmap v97: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 23 MiB/s wr, 984 op/s 2024-04-06T14:08:32.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:31 smithi134 ceph-mon[47549]: pgmap v97: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 23 MiB/s wr, 984 op/s 2024-04-06T14:08:33.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 165.0 s, 1993.6 tps, lat 50.045 ms stddev 56.692 2024-04-06T14:08:33.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:33 smithi105 ceph-mon[46969]: pgmap v98: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 19 MiB/s wr, 788 op/s 2024-04-06T14:08:34.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:33 smithi090 ceph-mon[43420]: pgmap v98: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 19 MiB/s wr, 788 op/s 2024-04-06T14:08:34.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:33 smithi134 ceph-mon[47549]: pgmap v98: 97 pgs: 97 active+clean; 4.7 GiB data, 6.4 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 19 MiB/s wr, 788 op/s 2024-04-06T14:08:35.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:35 smithi105 ceph-mon[46969]: pgmap v99: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 32 MiB/s wr, 1.33k op/s 2024-04-06T14:08:36.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:35 smithi090 ceph-mon[43420]: pgmap v99: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 32 MiB/s wr, 1.33k op/s 2024-04-06T14:08:36.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:35 smithi134 ceph-mon[47549]: pgmap v99: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 32 MiB/s wr, 1.33k op/s 2024-04-06T14:08:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 170.0 s, 2347.8 tps, lat 42.423 ms stddev 45.743 2024-04-06T14:08:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:38 smithi134 ceph-mon[47549]: pgmap v100: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 23 MiB/s wr, 909 op/s 2024-04-06T14:08:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:38 smithi105 ceph-mon[46969]: pgmap v100: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 23 MiB/s wr, 909 op/s 2024-04-06T14:08:39.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:38 smithi090 ceph-mon[43420]: pgmap v100: 97 pgs: 97 active+clean; 4.8 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 23 MiB/s wr, 909 op/s 2024-04-06T14:08:39.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:39 smithi105 ceph-mon[46969]: pgmap v101: 97 pgs: 97 active+clean; 4.9 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 29 MiB/s wr, 1.24k op/s 2024-04-06T14:08:40.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:39 smithi090 ceph-mon[43420]: pgmap v101: 97 pgs: 97 active+clean; 4.9 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 29 MiB/s wr, 1.24k op/s 2024-04-06T14:08:40.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:39 smithi134 ceph-mon[47549]: pgmap v101: 97 pgs: 97 active+clean; 4.9 GiB data, 6.7 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 29 MiB/s wr, 1.24k op/s 2024-04-06T14:08:41.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:41 smithi105 ceph-mon[46969]: pgmap v102: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 MiB/s wr, 1.34k op/s 2024-04-06T14:08:42.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:41 smithi090 ceph-mon[43420]: pgmap v102: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 MiB/s wr, 1.34k op/s 2024-04-06T14:08:42.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:41 smithi134 ceph-mon[47549]: pgmap v102: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 MiB/s wr, 1.34k op/s 2024-04-06T14:08:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 175.0 s, 948.7 tps, lat 106.264 ms stddev 175.035 2024-04-06T14:08:43.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:43 smithi105 ceph-mon[46969]: pgmap v103: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 25 MiB/s wr, 1.12k op/s 2024-04-06T14:08:44.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:43 smithi090 ceph-mon[43420]: pgmap v103: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 25 MiB/s wr, 1.12k op/s 2024-04-06T14:08:44.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:43 smithi134 ceph-mon[47549]: pgmap v103: 97 pgs: 97 active+clean; 4.9 GiB data, 6.8 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 25 MiB/s wr, 1.12k op/s 2024-04-06T14:08:45.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:45 smithi105 ceph-mon[46969]: pgmap v104: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 98 KiB/s rd, 53 MiB/s wr, 2.55k op/s 2024-04-06T14:08:46.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:45 smithi090 ceph-mon[43420]: pgmap v104: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 98 KiB/s rd, 53 MiB/s wr, 2.55k op/s 2024-04-06T14:08:46.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:45 smithi134 ceph-mon[47549]: pgmap v104: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 98 KiB/s rd, 53 MiB/s wr, 2.55k op/s 2024-04-06T14:08:48.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 180.0 s, 3014.4 tps, lat 32.988 ms stddev 36.763 2024-04-06T14:08:48.297 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:08:48.298 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:08:48.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.462+0000 7f657f7f5700 1 -- 172.21.15.90:0/3093788962 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 msgr2=0x7f6578108240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.462+0000 7f657f7f5700 1 --2- 172.21.15.90:0/3093788962 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 0x7f6578108240 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f65740098f0 tx=0x7f6574009c00 comp rx=0 tx=0).stop 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.464+0000 7f657f7f5700 1 -- 172.21.15.90:0/3093788962 shutdown_connections 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.464+0000 7f657f7f5700 1 --2- 172.21.15.90:0/3093788962 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f657810c6a0 0x7f6578114270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.464+0000 7f657f7f5700 1 --2- 172.21.15.90:0/3093788962 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 0x7f6578108240 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.464+0000 7f657f7f5700 1 --2- 172.21.15.90:0/3093788962 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6578108810 0x7f657810bfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.464+0000 7f657f7f5700 1 -- 172.21.15.90:0/3093788962 >> 172.21.15.90:0/3093788962 conn(0x7f6578076900 msgr2=0x7f6578078d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.466+0000 7f657f7f5700 1 -- 172.21.15.90:0/3093788962 shutdown_connections 2024-04-06T14:08:48.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.466+0000 7f657f7f5700 1 -- 172.21.15.90:0/3093788962 wait complete. 2024-04-06T14:08:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.466+0000 7f657f7f5700 1 Processor -- start 2024-04-06T14:08:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 -- start start 2024-04-06T14:08:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 0x7f65780836e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f657810c6a0 0x7f657807b6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65781b26e0 con 0x7f657810c6a0 2024-04-06T14:08:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65781b2850 con 0x7f657807bc30 2024-04-06T14:08:48.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.467+0000 7f657f7f5700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65781b2a10 con 0x7f6578107e60 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42050/0 (socket says 172.21.15.90:42050) 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 -- 172.21.15.90:0/481893516 learned_addr learned my addr 172.21.15.90:0/481893516 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 -- 172.21.15.90:0/481893516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 msgr2=0x7f65780836e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 0x7f65780836e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 -- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f657810c6a0 msgr2=0x7f657807b6f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f657810c6a0 0x7f657807b6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 -- 172.21.15.90:0/481893516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6574009520 con 0x7f657807bc30 2024-04-06T14:08:48.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f657dd92700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f657000b970 tx=0x7f657000bd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f656e7fc700 1 -- 172.21.15.90:0/481893516 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f657000c9b0 con 0x7f657807bc30 2024-04-06T14:08:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f656e7fc700 1 -- 172.21.15.90:0/481893516 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6570014440 con 0x7f657807bc30 2024-04-06T14:08:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.468+0000 7f656e7fc700 1 -- 172.21.15.90:0/481893516 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65700075d0 con 0x7f657807bc30 2024-04-06T14:08:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.469+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65781b2cf0 con 0x7f657807bc30 2024-04-06T14:08:48.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.469+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65781b31b0 con 0x7f657807bc30 2024-04-06T14:08:48.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.472+0000 7f656e7fc700 1 -- 172.21.15.90:0/481893516 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6570007730 con 0x7f657807bc30 2024-04-06T14:08:48.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.474+0000 7f656e7fc700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 0x7f65640a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.474+0000 7f656e7fc700 1 -- 172.21.15.90:0/481893516 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f6570006e40 con 0x7f657807bc30 2024-04-06T14:08:48.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.475+0000 7f658b9a9b80 1 Processor -- start 2024-04-06T14:08:48.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.475+0000 7f657d591700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 0x7f65640a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f657d591700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 0x7f65640a3850 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f6574009230 tx=0x7f6574009f90 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 -- start start 2024-04-06T14:08:48.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac381b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac38e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac2d9990 con 0x564dac38bde0 2024-04-06T14:08:48.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac2d7350 con 0x564dac3820a0 2024-04-06T14:08:48.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.476+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac2d2c40 con 0x564dac381780 2024-04-06T14:08:48.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.478+0000 7f657cd90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.478+0000 7f657cd90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42056/0 (socket says 172.21.15.90:42056) 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.478+0000 7f657cd90700 1 -- 172.21.15.90:0/994275475 learned_addr learned my addr 172.21.15.90:0/994275475 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.478+0000 7f657dd92700 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac38e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 -- 172.21.15.90:0/994275475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 msgr2=0x564dac381b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac381b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 -- 172.21.15.90:0/994275475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 msgr2=0x564dac38e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac38e1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 -- 172.21.15.90:0/994275475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564dac1543b0 con 0x564dac3820a0 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f657cd90700 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f6568009b80 tx=0x7f6568009e90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9e20c8b07c30cf8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f65637fe700 1 -- 172.21.15.90:0/994275475 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6568004230 con 0x564dac3820a0 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f65637fe700 1 -- 172.21.15.90:0/994275475 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f656800eda0 con 0x564dac3820a0 2024-04-06T14:08:48.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.479+0000 7f65637fe700 1 -- 172.21.15.90:0/994275475 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6568005020 con 0x564dac3820a0 2024-04-06T14:08:48.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.480+0000 7f658b9a9b80 1 -- 172.21.15.90:0/994275475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 msgr2=0x564dac38b8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.480+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f6568009b80 tx=0x7f6568009e90 comp rx=0 tx=0).stop 2024-04-06T14:08:48.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.485+0000 7f658b9a9b80 1 -- 172.21.15.90:0/994275475 shutdown_connections 2024-04-06T14:08:48.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.485+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac38e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.485+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac381b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.485+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/994275475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac38b8a0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.485+0000 7f658b9a9b80 1 -- 172.21.15.90:0/994275475 >> 172.21.15.90:0/994275475 conn(0x564dac2e06a0 msgr2=0x564dac37ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.487+0000 7f658b9a9b80 1 -- 172.21.15.90:0/994275475 shutdown_connections 2024-04-06T14:08:48.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.487+0000 7f658b9a9b80 1 -- 172.21.15.90:0/994275475 wait complete. 2024-04-06T14:08:48.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.487+0000 7f658b9a9b80 1 -- start start 2024-04-06T14:08:48.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.487+0000 7f658b9a9b80 1 Processor -- start 2024-04-06T14:08:48.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac3820a0 0x564dac388ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac389150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac26ed80 con 0x564dac3820a0 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac26ea20 con 0x564dac38bde0 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac1566a0 con 0x564dac381780 2024-04-06T14:08:48.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.488+0000 7f658b9a9b80 10 client.0 _ll_register_callbacks cb 0x564dac278470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:48.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37900/0 (socket says 172.21.15.90:37900) 2024-04-06T14:08:48.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 -- 172.21.15.90:0/127818652 learned_addr learned my addr 172.21.15.90:0/127818652 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657cd90700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac3820a0 0x564dac388ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 -- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 msgr2=0x564dac389150 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac389150 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 -- 172.21.15.90:0/127818652 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac3820a0 msgr2=0x564dac388ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac3820a0 0x564dac388ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.489+0000 7f657d591700 1 -- 172.21.15.90:0/127818652 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65680097f0 con 0x564dac381780 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f657d591700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f657400f6e0 tx=0x7f657400f710 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7aaa466c9743e894 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f6560ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6574028070 con 0x564dac381780 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f657400b770 con 0x564dac381780 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564dac293380 con 0x564dac381780 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f657401c070 con 0x564dac381780 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f658b9a9b80 10 client.25570 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:48.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564dac100b60 con 0x564dac381780 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.490+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6574039080 con 0x564dac381780 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f65740036a0 con 0x564dac381780 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 10 client.25570 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 10 client.25570 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 20 client.25570 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac39bae0 0x564dac39df00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 4 client.25570 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:48.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.491+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564dac2cb700 con 0x564dac39bae0 2024-04-06T14:08:48.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.496+0000 7f657cd90700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac39bae0 0x564dac39df00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.497+0000 7f657cd90700 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac39bae0 0x564dac39df00 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ac4ac58f10b80641 server_cookie=dd44bff9241f9e14 in_seq=0 out_seq=0 2024-04-06T14:08:48.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.497+0000 7f6560ff9700 10 client.25570 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:48.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.504+0000 7f6560ff9700 1 -- 172.21.15.90:0/127818652 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564dac2cb700 con 0x564dac39bae0 2024-04-06T14:08:48.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.504+0000 7f6560ff9700 10 client.25570 handle_command_reply: tid=0 2024-04-06T14:08:48.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.507+0000 7f658b9a9b80 1 client.25570 shutdown 2024-04-06T14:08:48.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.507+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 msgr2=0x564dac38b7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.507+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f657400f6e0 tx=0x7f657400f710 comp rx=0 tx=0).stop 2024-04-06T14:08:48.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 shutdown_connections 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac389150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac39bae0 0x564dac39df00 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac38b7e0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/127818652 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac3820a0 0x564dac388ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.511+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 >> 172.21.15.90:0/127818652 conn(0x564dac2e06a0 msgr2=0x564dac37bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.512+0000 7f6560ff9700 0 client.25570 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:48.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.512+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 shutdown_connections 2024-04-06T14:08:48.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.524+0000 7f658b9a9b80 1 -- 172.21.15.90:0/127818652 wait complete. 2024-04-06T14:08:48.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.533+0000 7f658b9a9b80 20 client.25570 trim_cache size 0 max 16384 2024-04-06T14:08:48.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.574+0000 7f658b9a9b80 1 Processor -- start 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 -- start start 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac392ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac3873a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac3878e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac389a60 con 0x564dac38bde0 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac387e20 con 0x564dac3820a0 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac3888a0 con 0x564dac381780 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f657cd90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac3873a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f657cd90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac3873a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42068/0 (socket says 172.21.15.90:42068) 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f657cd90700 1 -- 172.21.15.90:0/689089505 learned_addr learned my addr 172.21.15.90:0/689089505 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.577+0000 7f657d591700 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac392ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657dd92700 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac3878e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 -- 172.21.15.90:0/689089505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 msgr2=0x564dac3873a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac3873a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 -- 172.21.15.90:0/689089505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 msgr2=0x564dac3878e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac3878e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 -- 172.21.15.90:0/689089505 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564dac298ea0 con 0x564dac381780 2024-04-06T14:08:48.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.578+0000 7f657d591700 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac392ac0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f6574009810 tx=0x7f65740059e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=34e6694db27e141c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.582+0000 7f6563fff700 1 -- 172.21.15.90:0/689089505 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f657403c070 con 0x564dac381780 2024-04-06T14:08:48.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.582+0000 7f6563fff700 1 -- 172.21.15.90:0/689089505 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564dac293380 con 0x564dac381780 2024-04-06T14:08:48.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.582+0000 7f6563fff700 1 -- 172.21.15.90:0/689089505 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6574028040 con 0x564dac381780 2024-04-06T14:08:48.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.585+0000 7f658b9a9b80 1 -- 172.21.15.90:0/689089505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 msgr2=0x564dac392ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.585+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac392ac0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f6574009810 tx=0x7f65740059e0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.586+0000 7f658b9a9b80 1 -- 172.21.15.90:0/689089505 shutdown_connections 2024-04-06T14:08:48.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.586+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac38bde0 0x564dac3878e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.586+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac381780 0x564dac392ac0 secure :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f6574009810 tx=0x7f65740059e0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.586+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/689089505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac3820a0 0x564dac3873a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.586+0000 7f658b9a9b80 1 -- 172.21.15.90:0/689089505 >> 172.21.15.90:0/689089505 conn(0x564dac2e06a0 msgr2=0x564dac383da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.592+0000 7f658b9a9b80 1 -- 172.21.15.90:0/689089505 shutdown_connections 2024-04-06T14:08:48.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.593+0000 7f658b9a9b80 1 -- 172.21.15.90:0/689089505 wait complete. 2024-04-06T14:08:48.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 -- start start 2024-04-06T14:08:48.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 Processor -- start 2024-04-06T14:08:48.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac3917e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac391d20 0x564dac3ef920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac393040 con 0x564dac391d20 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac3efe60 con 0x564dac38bde0 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564dac3efff0 con 0x564dac3820a0 2024-04-06T14:08:48.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.594+0000 7f658b9a9b80 10 client.0 _ll_register_callbacks cb 0x564dac278470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657cd90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac3917e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37920/0 (socket says 172.21.15.90:37920) 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 -- 172.21.15.90:0/1912737359 learned_addr learned my addr 172.21.15.90:0/1912737359 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 -- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 msgr2=0x564dac3917e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac3917e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 -- 172.21.15.90:0/1912737359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac391d20 msgr2=0x564dac3ef920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac391d20 0x564dac3ef920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.597+0000 7f657d591700 1 -- 172.21.15.90:0/1912737359 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564dac100b60 con 0x564dac3820a0 2024-04-06T14:08:48.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.598+0000 7f657d591700 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f6574005cd0 tx=0x7f6574005320 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b7e4ba77338bd6dd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.598+0000 7f6560ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:08:48.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.598+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f657403c050 con 0x564dac3820a0 2024-04-06T14:08:48.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.598+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f657400bb80 con 0x564dac3820a0 2024-04-06T14:08:48.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.600+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f656800eda0 con 0x564dac3820a0 2024-04-06T14:08:48.635 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:48 smithi090 ceph-mon[43420]: pgmap v105: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 39 MiB/s wr, 2.01k op/s 2024-04-06T14:08:48.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.639+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6574028040 con 0x564dac3820a0 2024-04-06T14:08:48.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.643+0000 7f658b9a9b80 10 client.25576 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:48.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.643+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564dac158cd0 con 0x564dac3820a0 2024-04-06T14:08:48.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f656800eda0 con 0x564dac3820a0 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6574048070 con 0x564dac3820a0 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 10 client.25576 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 10 client.25576 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 20 client.25576 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac3f1320 0x564dac3f3710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 4 client.25576 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x564dac121cf0 con 0x564dac3f1320 2024-04-06T14:08:48.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.645+0000 7f657cd90700 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac3f1320 0x564dac3f3710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f6560ff9700 10 client.25576 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:48.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f657cd90700 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac3f1320 0x564dac3f3710 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=66648d560d658e22 server_cookie=96644fcbcdd6dac7 in_seq=0 out_seq=0 2024-04-06T14:08:48.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f6560ff9700 1 -- 172.21.15.90:0/1912737359 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x564dac121cf0 con 0x564dac3f1320 2024-04-06T14:08:48.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f6560ff9700 10 client.25576 handle_command_reply: tid=0 2024-04-06T14:08:48.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 client.25576 shutdown 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 msgr2=0x564dac3912a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f6574005cd0 tx=0x7f6574005320 comp rx=0 tx=0).stop 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 shutdown_connections 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564dac391d20 0x564dac3ef920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564dac3f1320 0x564dac3f3710 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564dac3820a0 0x564dac3912a0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f6560ff9700 0 client.25576 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 --2- 172.21.15.90:0/1912737359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564dac38bde0 0x564dac3917e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 >> 172.21.15.90:0/1912737359 conn(0x564dac2e06a0 msgr2=0x564dac3872e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 shutdown_connections 2024-04-06T14:08:48.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.646+0000 7f658b9a9b80 1 -- 172.21.15.90:0/1912737359 wait complete. 2024-04-06T14:08:48.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f658b9a9b80 20 client.25576 trim_cache size 0 max 16384 2024-04-06T14:08:48.649 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:48.649 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:08:48.649 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:08:48.649 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 msgr2=0x7f65640a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 0x7f65640a3850 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f6574009230 tx=0x7f6574009f90 comp rx=0 tx=0).stop 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 msgr2=0x7f657807c0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f657000b970 tx=0x7f657000bd30 comp rx=0 tx=0).stop 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 shutdown_connections 2024-04-06T14:08:48.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65640a1390 0x7f65640a3850 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f657807bc30 0x7f657807c0e0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6578107e60 0x7f65780836e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 --2- 172.21.15.90:0/481893516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f657810c6a0 0x7f657807b6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 >> 172.21.15.90:0/481893516 conn(0x7f6578076900 msgr2=0x7f6578078cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 shutdown_connections 2024-04-06T14:08:48.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.647+0000 7f657f7f5700 1 -- 172.21.15.90:0/481893516 wait complete. 2024-04-06T14:08:48.665 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:08:48.666 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:08:48.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 -- 172.21.15.90:0/3811027014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd2781080f0 msgr2=0x7fd27810bec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3811027014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd2781080f0 0x7fd27810bec0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fd2680099c0 tx=0x7fd268009cd0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 -- 172.21.15.90:0/3811027014 shutdown_connections 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3811027014 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd27810c5f0 0x7fd278114170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3811027014 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd278107740 0x7fd278107b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.842+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3811027014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd2781080f0 0x7fd27810bec0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.843+0000 7fd27f34f700 1 -- 172.21.15.90:0/3811027014 >> 172.21.15.90:0/3811027014 conn(0x7fd278076c50 msgr2=0x7fd278105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:48.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.844+0000 7fd27f34f700 1 -- 172.21.15.90:0/3811027014 shutdown_connections 2024-04-06T14:08:48.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.845+0000 7fd27f34f700 1 -- 172.21.15.90:0/3811027014 wait complete. 2024-04-06T14:08:48.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.846+0000 7fd27f34f700 1 Processor -- start 2024-04-06T14:08:48.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.846+0000 7fd27f34f700 1 -- start start 2024-04-06T14:08:48.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd27810c5f0 0x7fd2781a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd27819d5b0 0x7fd27819da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd27819e000 con 0x7fd27819d5b0 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd27819e140 con 0x7fd27810c5f0 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27f34f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd27819e300 con 0x7fd278107740 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37936/0 (socket says 172.21.15.90:37936) 2024-04-06T14:08:48.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 -- 172.21.15.90:0/3737011409 learned_addr learned my addr 172.21.15.90:0/3737011409 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:48.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d8ec700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd27819d5b0 0x7fd27819da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 -- 172.21.15.90:0/3737011409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd27810c5f0 msgr2=0x7fd2781a3960 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd27810c5f0 0x7fd2781a3960 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 -- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd27819d5b0 msgr2=0x7fd27819da30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd27819d5b0 0x7fd27819da30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.847+0000 7fd27d0eb700 1 -- 172.21.15.90:0/3737011409 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd270009c40 con 0x7fd278107740 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd27d0eb700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fd27400d980 tx=0x7fd27400dd40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd27400bb60 con 0x7fd278107740 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd27400de60 con 0x7fd278107740 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd274004e60 con 0x7fd278107740 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd2680095f0 con 0x7fd278107740 2024-04-06T14:08:48.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.848+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd278079840 con 0x7fd278107740 2024-04-06T14:08:48.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.849+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd27810f8c0 con 0x7fd278107740 2024-04-06T14:08:48.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.851+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fd274004020 con 0x7fd278107740 2024-04-06T14:08:48.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.852+0000 7fd26e7fc700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 0x7fd2640a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:48.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.852+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fd2740ccce0 con 0x7fd278107740 2024-04-06T14:08:48.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.852+0000 7fd27c8ea700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 0x7fd2640a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:48.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.854+0000 7fd27c8ea700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 0x7fd2640a38e0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd268009fd0 tx=0x7fd268000bc0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:48.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:48.855+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fd2740961c0 con 0x7fd278107740 2024-04-06T14:08:48.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:48 smithi134 ceph-mon[47549]: pgmap v105: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 39 MiB/s wr, 2.01k op/s 2024-04-06T14:08:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:48 smithi105 ceph-mon[46969]: pgmap v105: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 39 MiB/s wr, 2.01k op/s 2024-04-06T14:08:49.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.253+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fd2781089a0 con 0x7fd278107740 2024-04-06T14:08:49.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.254+0000 7fd26e7fc700 1 -- 172.21.15.90:0/3737011409 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fd274016030 con 0x7fd278107740 2024-04-06T14:08:49.255 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:08:49.255 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:08:49.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 msgr2=0x7fd2640a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 0x7fd2640a38e0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fd268009fd0 tx=0x7fd268000bc0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 msgr2=0x7fd2781a3420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7fd27400d980 tx=0x7fd27400dd40 comp rx=0 tx=0).stop 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 shutdown_connections 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd2640a1420 0x7fd2640a38e0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd27819d5b0 0x7fd27819da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd278107740 0x7fd2781a3420 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 --2- 172.21.15.90:0/3737011409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd27810c5f0 0x7fd2781a3960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 >> 172.21.15.90:0/3737011409 conn(0x7fd278076c50 msgr2=0x7fd278078810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 shutdown_connections 2024-04-06T14:08:49.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.257+0000 7fd27f34f700 1 -- 172.21.15.90:0/3737011409 wait complete. 2024-04-06T14:08:49.261 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:08:49.277 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:49.278 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:08:49.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.470+0000 7f843e9b8700 1 -- 172.21.15.90:0/3740863627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8438107590 msgr2=0x7f8438107970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.470+0000 7f843e9b8700 1 --2- 172.21.15.90:0/3740863627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8438107590 0x7f8438107970 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f842c009990 tx=0x7f842c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.474+0000 7f843e9b8700 1 -- 172.21.15.90:0/3740863627 shutdown_connections 2024-04-06T14:08:49.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.474+0000 7f843e9b8700 1 --2- 172.21.15.90:0/3740863627 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843810c6b0 0x7f8438114230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.474+0000 7f843e9b8700 1 --2- 172.21.15.90:0/3740863627 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8438107590 0x7f8438107970 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.474+0000 7f843e9b8700 1 --2- 172.21.15.90:0/3740863627 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8438107f40 0x7f843810bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.474+0000 7f843e9b8700 1 -- 172.21.15.90:0/3740863627 >> 172.21.15.90:0/3740863627 conn(0x7f84380752a0 msgr2=0x7f84380776c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.475+0000 7f843e9b8700 1 -- 172.21.15.90:0/3740863627 shutdown_connections 2024-04-06T14:08:49.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.475+0000 7f843e9b8700 1 -- 172.21.15.90:0/3740863627 wait complete. 2024-04-06T14:08:49.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.476+0000 7f843e9b8700 1 Processor -- start 2024-04-06T14:08:49.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.476+0000 7f843e9b8700 1 -- start start 2024-04-06T14:08:49.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8438107590 0x7f84381a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 0x7f843819dce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f843819e2b0 con 0x7f8438107590 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f843819e3f0 con 0x7f843810c6b0 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843e9b8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f843819e5b0 con 0x7f843819d860 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843cf55700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 0x7f843819dce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42116/0 (socket says 172.21.15.90:42116) 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f843cf55700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 0x7f843819dce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37954/0 (socket says 172.21.15.90:37954) 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 -- 172.21.15.90:0/1408330419 learned_addr learned my addr 172.21.15.90:0/1408330419 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 -- 172.21.15.90:0/1408330419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 msgr2=0x7f843819dce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 0x7f843819dce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 -- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8438107590 msgr2=0x7f84381a36d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8438107590 0x7f84381a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.477+0000 7f84377fe700 1 -- 172.21.15.90:0/1408330419 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f842c0095f0 con 0x7f843810c6b0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f84377fe700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f843000b9e0 tx=0x7f843000bcf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f84357fa700 1 -- 172.21.15.90:0/1408330419 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f843000c9b0 con 0x7f843810c6b0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f84357fa700 1 -- 172.21.15.90:0/1408330419 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8430014440 con 0x7f843810c6b0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84381aa5f0 con 0x7f843810c6b0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f84357fa700 1 -- 172.21.15.90:0/1408330419 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8430012580 con 0x7f843810c6b0 2024-04-06T14:08:49.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.478+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84381aaad0 con 0x7f843810c6b0 2024-04-06T14:08:49.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.482+0000 7f844ab6cb80 1 Processor -- start 2024-04-06T14:08:49.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.482+0000 7f844ab6cb80 1 -- start start 2024-04-06T14:08:49.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403128b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031351d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403080990 con 0x55d403132de0 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d40307e350 con 0x55d4031290a0 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403079c40 con 0x55d403128780 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f8437fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f8437fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42128/0 (socket says 172.21.15.90:42128) 2024-04-06T14:08:49.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.483+0000 7f8437fff700 1 -- 172.21.15.90:0/1865734397 learned_addr learned my addr 172.21.15.90:0/1865734397 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f843cf55700 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403128b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 -- 172.21.15.90:0/1865734397 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 msgr2=0x55d403128b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403128b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 -- 172.21.15.90:0/1865734397 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 msgr2=0x55d4031351d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031351d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 -- 172.21.15.90:0/1865734397 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d402efb3b0 con 0x55d4031290a0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f84357fa700 1 -- 172.21.15.90:0/1408330419 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f84300126e0 con 0x7f843810c6b0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f8437fff700 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f842c005f50 tx=0x7f842c009f20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9ea9e2b6cc246e6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.484+0000 7f84267fc700 1 -- 172.21.15.90:0/1865734397 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f842c01d070 con 0x55d4031290a0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f84267fc700 1 -- 172.21.15.90:0/1865734397 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f842c00be00 con 0x55d4031290a0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f84267fc700 1 -- 172.21.15.90:0/1865734397 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f842c005420 con 0x55d4031290a0 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f84357fa700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 0x7f84200a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f84357fa700 1 -- 172.21.15.90:0/1408330419 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f84300cc2b0 con 0x7f843810c6b0 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f843cf55700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 0x7f84200a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1865734397 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 msgr2=0x55d4031328a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f842c005f50 tx=0x7f842c009f20 comp rx=0 tx=0).stop 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1865734397 shutdown_connections 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.485+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031351d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403128b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1865734397 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d4031328a0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1865734397 >> 172.21.15.90:0/1865734397 conn(0x55d4030876a0 msgr2=0x55d403123a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1865734397 shutdown_connections 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1865734397 wait complete. 2024-04-06T14:08:49.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f843cf55700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 0x7f84200a3890 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f84280099a0 tx=0x7f8428008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 -- start start 2024-04-06T14:08:49.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.486+0000 7f844ab6cb80 1 Processor -- start 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403132840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031303b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403015d80 con 0x55d403132de0 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403015a20 con 0x55d4031290a0 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d402efd6a0 con 0x55d403128780 2024-04-06T14:08:49.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f844ab6cb80 10 client.0 _ll_register_callbacks cb 0x55d40301f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42134/0 (socket says 172.21.15.90:42134) 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 -- 172.21.15.90:0/1282190607 learned_addr learned my addr 172.21.15.90:0/1282190607 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 -- 172.21.15.90:0/1282190607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 msgr2=0x55d403132840 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403132840 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 -- 172.21.15.90:0/1282190607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 msgr2=0x55d4031303b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f843cf55700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031303b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031303b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 -- 172.21.15.90:0/1282190607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8438114aa0 con 0x55d4031290a0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.487+0000 7f84377fe700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f843000be10 tx=0x7f84300a1920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f51de78fa8272b8b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84300c9e60 con 0x55d4031290a0 2024-04-06T14:08:49.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8430098b50 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d40303a380 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8430098d10 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f844ab6cb80 10 client.25670 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d402ea7b60 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d40303a380 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8430095450 con 0x55d4031290a0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f844ab6cb80 10 client.25670 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.488+0000 7f844ab6cb80 10 client.25670 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f844ab6cb80 20 client.25670 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031539e0 0x55d403155e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f844ab6cb80 4 client.25670 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d403072700 con 0x55d4031539e0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f8437fff700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031539e0 0x55d403155e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f841b7fe700 10 client.25670 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:49.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.489+0000 7f8437fff700 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031539e0 0x55d403155e00 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=fbea445fb2b8b84a server_cookie=93caba6bf6018a5c in_seq=0 out_seq=0 2024-04-06T14:08:49.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.490+0000 7f841b7fe700 1 -- 172.21.15.90:0/1282190607 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d403072700 con 0x55d4031539e0 2024-04-06T14:08:49.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.490+0000 7f841b7fe700 10 client.25670 handle_command_reply: tid=0 2024-04-06T14:08:49.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.490+0000 7f844ab6cb80 1 client.25670 shutdown 2024-04-06T14:08:49.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 msgr2=0x55d40312fc70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f843000be10 tx=0x7f84300a1920 comp rx=0 tx=0).stop 2024-04-06T14:08:49.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 shutdown_connections 2024-04-06T14:08:49.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031539e0 0x55d403155e00 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403132de0 0x55d4031303b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403128780 0x55d403132840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1282190607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4031290a0 0x55d40312fc70 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f841b7fe700 0 client.25670 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 >> 172.21.15.90:0/1282190607 conn(0x55d4030876a0 msgr2=0x55d403122c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 shutdown_connections 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1282190607 wait complete. 2024-04-06T14:08:49.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.493+0000 7f844ab6cb80 20 client.25670 trim_cache size 0 max 16384 2024-04-06T14:08:49.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 Processor -- start 2024-04-06T14:08:49.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 -- start start 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 0x55d40312fc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403132de0 0x55d4031233a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d40312a290 con 0x55d4031290a0 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403124dc0 con 0x55d403128780 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.509+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403139f70 con 0x55d403132de0 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.510+0000 7f843cf55700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403132de0 0x55d4031233a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.510+0000 7f84377fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.510+0000 7f84377fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59402/0 (socket says 172.21.15.90:59402) 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.510+0000 7f84377fe700 1 -- 172.21.15.90:0/1431905059 learned_addr learned my addr 172.21.15.90:0/1431905059 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.510+0000 7f8437fff700 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 0x55d40312fc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 -- 172.21.15.90:0/1431905059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403132de0 msgr2=0x55d4031233a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403132de0 0x55d4031233a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 -- 172.21.15.90:0/1431905059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 msgr2=0x55d403139a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 -- 172.21.15.90:0/1431905059 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d40303fea0 con 0x55d403128780 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f84377fe700 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:08:49.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.511+0000 7f8437fff700 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 0x55d40312fc10 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f842c009990 tx=0x7f842c000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=21a718f23c11b1c5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.512+0000 7f8426ffd700 1 -- 172.21.15.90:0/1431905059 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f842c01d070 con 0x55d403128780 2024-04-06T14:08:49.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.512+0000 7f8426ffd700 1 -- 172.21.15.90:0/1431905059 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f842c01e040 con 0x55d403128780 2024-04-06T14:08:49.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.512+0000 7f8426ffd700 1 -- 172.21.15.90:0/1431905059 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f842c0054f0 con 0x55d403128780 2024-04-06T14:08:49.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1431905059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 msgr2=0x55d40312fc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 0x55d40312fc10 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f842c009990 tx=0x7f842c000bc0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1431905059 shutdown_connections 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d403132de0 0x55d4031233a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403128780 0x55d40312fc10 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1431905059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4031290a0 0x55d403139a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.513+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1431905059 >> 172.21.15.90:0/1431905059 conn(0x55d4030876a0 msgr2=0x55d40312f500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.514+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1431905059 shutdown_connections 2024-04-06T14:08:49.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1431905059 wait complete. 2024-04-06T14:08:49.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 -- start start 2024-04-06T14:08:49.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 Processor -- start 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4031290a0 0x55d403139840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403132de0 0x55d403137a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4031380e0 con 0x55d403128780 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.515+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403138250 con 0x55d403132de0 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f844ab6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d403138410 con 0x55d4031290a0 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f844ab6cb80 10 client.0 _ll_register_callbacks cb 0x55d40301f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:49.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f84377fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4031290a0 0x55d403139840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59416/0 (socket says 172.21.15.90:59416) 2024-04-06T14:08:49.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 -- 172.21.15.90:0/1985909418 learned_addr learned my addr 172.21.15.90:0/1985909418 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 -- 172.21.15.90:0/1985909418 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4031290a0 msgr2=0x55d403139840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4031290a0 0x55d403139840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 -- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403132de0 msgr2=0x55d403137a10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403132de0 0x55d403137a10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 -- 172.21.15.90:0/1985909418 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d402ea7b60 con 0x55d403128780 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f8437fff700 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f842c0037c0 tx=0x7f842c009290 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6dac6a0eee52a01 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.516+0000 7f841b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f842c01d040 con 0x55d403128780 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f842c00baa0 con 0x55d403128780 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f842c00be00 con 0x55d403128780 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f842c003a60 con 0x55d403128780 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f844ab6cb80 10 client.15990 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:49.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f842c02a730 con 0x55d403128780 2024-04-06T14:08:49.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f842c00be00 con 0x55d403128780 2024-04-06T14:08:49.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.517+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f842c005580 con 0x55d403128780 2024-04-06T14:08:49.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 10 client.15990 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:49.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 10 client.15990 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 20 client.15990 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031b9510 0x55d4031bb900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 4 client.15990 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d402ec8cf0 con 0x55d4031b9510 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f84377fe700 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031b9510 0x55d4031bb900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.518+0000 7f84377fe700 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031b9510 0x55d4031bb900 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=ab48bfda3d36bfe6 server_cookie=544cfba54307cb69 in_seq=0 out_seq=0 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.519+0000 7f841b7fe700 10 client.15990 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.519+0000 7f841b7fe700 1 -- 172.21.15.90:0/1985909418 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d402ec8cf0 con 0x55d4031b9510 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.519+0000 7f841b7fe700 10 client.15990 handle_command_reply: tid=0 2024-04-06T14:08:49.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.519+0000 7f844ab6cb80 1 client.15990 shutdown 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.520+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 msgr2=0x55d403139300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.520+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 secure :-1 s=READY pgs=518 cs=0 l=1 rev1=1 crypto rx=0x7f842c0037c0 tx=0x7f842c009290 comp rx=0 tx=0).stop 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 shutdown_connections 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55d4031b9510 0x55d4031bb900 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d403132de0 0x55d403137a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f841b7fe700 0 client.15990 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d403128780 0x55d403139300 unknown :-1 s=CLOSED pgs=518 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 --2- 172.21.15.90:0/1985909418 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4031290a0 0x55d403139840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 >> 172.21.15.90:0/1985909418 conn(0x55d4030876a0 msgr2=0x55d403128000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 shutdown_connections 2024-04-06T14:08:49.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 1 -- 172.21.15.90:0/1985909418 wait complete. 2024-04-06T14:08:49.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.521+0000 7f844ab6cb80 20 client.15990 trim_cache size 0 max 16384 2024-04-06T14:08:49.522 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.522+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 msgr2=0x7f84200a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.522+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 0x7f84200a3890 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f84280099a0 tx=0x7f8428008040 comp rx=0 tx=0).stop 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.522+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 msgr2=0x7f84381a3c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.522+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f843000b9e0 tx=0x7f843000bcf0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 shutdown_connections 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f84200a13d0 0x7f84200a3890 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f843819d860 0x7f843819dce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8438107590 0x7f84381a36d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 --2- 172.21.15.90:0/1408330419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f843810c6b0 0x7f84381a3c10 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 >> 172.21.15.90:0/1408330419 conn(0x7f84380752a0 msgr2=0x7f8438076640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 shutdown_connections 2024-04-06T14:08:49.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.523+0000 7f843e9b8700 1 -- 172.21.15.90:0/1408330419 wait complete. 2024-04-06T14:08:49.540 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:49.540 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 -- 172.21.15.90:0/1410515944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 msgr2=0x7f039c109b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 --2- 172.21.15.90:0/1410515944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c109b30 secure :-1 s=READY pgs=519 cs=0 l=1 rev1=1 crypto rx=0x7f038c0099c0 tx=0x7f038c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 -- 172.21.15.90:0/1410515944 shutdown_connections 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 --2- 172.21.15.90:0/1410515944 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c112d00 0x7f039c1150f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 --2- 172.21.15.90:0/1410515944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c109b30 unknown :-1 s=CLOSED pgs=519 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 --2- 172.21.15.90:0/1410515944 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c10a070 0x7f039c1127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 -- 172.21.15.90:0/1410515944 >> 172.21.15.90:0/1410515944 conn(0x7f039c06c250 msgr2=0x7f039c06c660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 -- 172.21.15.90:0/1410515944 shutdown_connections 2024-04-06T14:08:49.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.714+0000 7f03a2de1700 1 -- 172.21.15.90:0/1410515944 wait complete. 2024-04-06T14:08:49.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.715+0000 7f03a2de1700 1 Processor -- start 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 -- start start 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c1a5590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c10a070 0x7f039c1a5ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f039c19e1d0 con 0x7f039c109750 2024-04-06T14:08:49.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f039c19e340 con 0x7f039c112d00 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a2de1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f039c19e500 con 0x7f039c10a070 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f039bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c10a070 0x7f039c1a5ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a137e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a137e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42182/0 (socket says 172.21.15.90:42182) 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.716+0000 7f03a137e700 1 -- 172.21.15.90:0/4280518194 learned_addr learned my addr 172.21.15.90:0/4280518194 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c1a5590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 -- 172.21.15.90:0/4280518194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c10a070 msgr2=0x7f039c1a5ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c10a070 0x7f039c1a5ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 -- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 msgr2=0x7f039c1a5590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c1a5590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 -- 172.21.15.90:0/4280518194 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f038c0095f0 con 0x7f039c112d00 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f03a137e700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f039400ef30 tx=0x7f039400c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f0399ffb700 1 -- 172.21.15.90:0/4280518194 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0394018070 con 0x7f039c112d00 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f0399ffb700 1 -- 172.21.15.90:0/4280518194 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f039400f040 con 0x7f039c112d00 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.717+0000 7f0399ffb700 1 -- 172.21.15.90:0/4280518194 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f039400cd00 con 0x7f039c112d00 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.718+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f039c19e780 con 0x7f039c112d00 2024-04-06T14:08:49.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.718+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f039c1bb420 con 0x7f039c112d00 2024-04-06T14:08:49.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.720+0000 7f0399ffb700 1 -- 172.21.15.90:0/4280518194 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0394003a00 con 0x7f039c112d00 2024-04-06T14:08:49.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.721+0000 7f0399ffb700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 0x7f03840a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.721+0000 7f0399ffb700 1 -- 172.21.15.90:0/4280518194 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f03940cc7d0 con 0x7f039c112d00 2024-04-06T14:08:49.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.722+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 0x7f03840a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.722+0000 7f03aef95b80 1 Processor -- start 2024-04-06T14:08:49.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 -- start start 2024-04-06T14:08:49.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 0x7f03840a3750 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f038c009990 tx=0x7f038c019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea3780 0x560f0eea3b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 0x560f0eead8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeb01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0edfb990 con 0x560f0eea40a0 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0edf9350 con 0x560f0eeadde0 2024-04-06T14:08:49.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03aef95b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0edf4c40 con 0x560f0eea3780 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f039bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 0x560f0eead8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f039bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 0x560f0eead8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59440/0 (socket says 172.21.15.90:59440) 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f039bfff700 1 -- 172.21.15.90:0/3178929774 learned_addr learned my addr 172.21.15.90:0/3178929774 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.723+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea3780 0x560f0eea3b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeb01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 -- 172.21.15.90:0/3178929774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea3780 msgr2=0x560f0eea3b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea3780 0x560f0eea3b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 -- 172.21.15.90:0/3178929774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 msgr2=0x560f0eead8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 0x560f0eead8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 -- 172.21.15.90:0/3178929774 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560f0ec763b0 con 0x560f0eeadde0 2024-04-06T14:08:49.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.724+0000 7f03a137e700 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeb01d0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f03940c9ef0 tx=0x7f0394099800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5d411a114529e3d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.726+0000 7f0382ffd700 1 -- 172.21.15.90:0/3178929774 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03940b7ea0 con 0x560f0eeadde0 2024-04-06T14:08:49.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.726+0000 7f0382ffd700 1 -- 172.21.15.90:0/3178929774 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0394084040 con 0x560f0eeadde0 2024-04-06T14:08:49.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.726+0000 7f0382ffd700 1 -- 172.21.15.90:0/3178929774 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f039409d770 con 0x560f0eeadde0 2024-04-06T14:08:49.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 -- 172.21.15.90:0/3178929774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 msgr2=0x560f0eeb01d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeb01d0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f03940c9ef0 tx=0x7f0394099800 comp rx=0 tx=0).stop 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 -- 172.21.15.90:0/3178929774 shutdown_connections 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeb01d0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea3780 0x560f0eea3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 --2- 172.21.15.90:0/3178929774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea40a0 0x560f0eead8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.727+0000 7f03aef95b80 1 -- 172.21.15.90:0/3178929774 >> 172.21.15.90:0/3178929774 conn(0x560f0ee026a0 msgr2=0x560f0ee9ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.728+0000 7f03aef95b80 1 -- 172.21.15.90:0/3178929774 shutdown_connections 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.728+0000 7f03aef95b80 1 -- 172.21.15.90:0/3178929774 wait complete. 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.728+0000 7f03aef95b80 1 -- start start 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 Processor -- start 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeaac60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0eeab1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ed90d80 con 0x560f0eea3780 2024-04-06T14:08:49.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ed90a20 con 0x560f0eeadde0 2024-04-06T14:08:49.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ec786a0 con 0x560f0eea40a0 2024-04-06T14:08:49.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03aef95b80 10 client.0 _ll_register_callbacks cb 0x560f0ed9a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:49.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03a137e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03a137e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42194/0 (socket says 172.21.15.90:42194) 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.729+0000 7f03a137e700 1 -- 172.21.15.90:0/2496514437 learned_addr learned my addr 172.21.15.90:0/2496514437 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 -- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 msgr2=0x560f0eeab1a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0eeab1a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 -- 172.21.15.90:0/2496514437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 msgr2=0x560f0eeaac60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeaac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 -- 172.21.15.90:0/2496514437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f039c1bb420 con 0x560f0eeadde0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03a137e700 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0394099640 tx=0x7f0394099e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1df13661168dfd0d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f037bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f03940b7ea0 con 0x560f0eeadde0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560f0edb5380 con 0x560f0eeadde0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f039400ce60 con 0x560f0eeadde0 2024-04-06T14:08:49.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.730+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f03940b2260 con 0x560f0eeadde0 2024-04-06T14:08:49.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 10 client.25685 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560f0ec22b60 con 0x560f0eeadde0 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f039407c080 con 0x560f0eeadde0 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0394065170 con 0x560f0eeadde0 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 10 client.25685 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 10 client.25685 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 20 client.25685 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0eebd5e0 0x560f0eebf9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 4 client.25685 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.731+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560f0eded700 con 0x560f0eebd5e0 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.732+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0eebd5e0 0x560f0eebf9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.732+0000 7f037bfff700 10 client.25685 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:49.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.732+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0eebd5e0 0x560f0eebf9d0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fd0a2160d4061ebb server_cookie=7dceaa461ee1152c in_seq=0 out_seq=0 2024-04-06T14:08:49.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f037bfff700 1 -- 172.21.15.90:0/2496514437 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560f0eded700 con 0x560f0eebd5e0 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f037bfff700 10 client.25685 handle_command_reply: tid=0 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 client.25685 shutdown 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 msgr2=0x560f0eeab6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0394099640 tx=0x7f0394099e80 comp rx=0 tx=0).stop 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 shutdown_connections 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0eebd5e0 0x560f0eebf9d0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eeadde0 0x560f0eeab6e0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeaac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f037bfff700 0 client.25685 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2496514437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0eeab1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 >> 172.21.15.90:0/2496514437 conn(0x560f0ee026a0 msgr2=0x560f0ee9dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 shutdown_connections 2024-04-06T14:08:49.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 1 -- 172.21.15.90:0/2496514437 wait complete. 2024-04-06T14:08:49.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.733+0000 7f03aef95b80 20 client.25685 trim_cache size 0 max 16384 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.753+0000 7f03aef95b80 1 Processor -- start 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.753+0000 7f03aef95b80 1 -- start start 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0ee9e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eeadde0 0x560f0eea7ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0eeac1c0 con 0x560f0eeadde0 2024-04-06T14:08:49.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ee9ff80 con 0x560f0eea3780 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03aef95b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ee963c0 con 0x560f0eea40a0 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a137e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eeadde0 0x560f0eea7ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f039bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0ee9e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42196/0 (socket says 172.21.15.90:42196) 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f039bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0ee9e8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38026/0 (socket says 172.21.15.90:38026) 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 -- 172.21.15.90:0/1002172898 learned_addr learned my addr 172.21.15.90:0/1002172898 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 -- 172.21.15.90:0/1002172898 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 msgr2=0x560f0ee9e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0ee9e8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 -- 172.21.15.90:0/1002172898 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eeadde0 msgr2=0x560f0eea7ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eeadde0 0x560f0eea7ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.754+0000 7f03a0b7d700 1 -- 172.21.15.90:0/1002172898 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560f0edbaea0 con 0x560f0eea3780 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f038c005cd0 tx=0x7f038c009cd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bc3f4e594f6740a8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03837fe700 1 -- 172.21.15.90:0/1002172898 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f038c00f940 con 0x560f0eea3780 2024-04-06T14:08:49.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03837fe700 1 -- 172.21.15.90:0/1002172898 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f038c01d040 con 0x560f0eea3780 2024-04-06T14:08:49.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03837fe700 1 -- 172.21.15.90:0/1002172898 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f038c02fea0 con 0x560f0eea3780 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03aef95b80 1 -- 172.21.15.90:0/1002172898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 msgr2=0x560f0ee9e3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.755+0000 7f03aef95b80 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f038c005cd0 tx=0x7f038c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 -- 172.21.15.90:0/1002172898 shutdown_connections 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eeadde0 0x560f0eea7ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea3780 0x560f0ee9e3a0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 --2- 172.21.15.90:0/1002172898 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eea40a0 0x560f0ee9e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 -- 172.21.15.90:0/1002172898 >> 172.21.15.90:0/1002172898 conn(0x560f0ee026a0 msgr2=0x560f0eea9530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 -- 172.21.15.90:0/1002172898 shutdown_connections 2024-04-06T14:08:49.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 -- 172.21.15.90:0/1002172898 wait complete. 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 -- start start 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.756+0000 7f03aef95b80 1 Processor -- start 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea40a0 0x560f0eeb30e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eeadde0 0x560f0eeb3820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0eeb3f30 con 0x560f0eea3780 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ef113a0 con 0x560f0eea40a0 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560f0ef114e0 con 0x560f0eeadde0 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03aef95b80 10 client.0 _ll_register_callbacks cb 0x560f0ed9a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59460/0 (socket says 172.21.15.90:59460) 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 -- 172.21.15.90:0/2478927001 learned_addr learned my addr 172.21.15.90:0/2478927001 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a137e700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eeadde0 0x560f0eeb3820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 -- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eeadde0 msgr2=0x560f0eeb3820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eeadde0 0x560f0eeb3820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 -- 172.21.15.90:0/2478927001 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea40a0 msgr2=0x560f0eeb30e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea40a0 0x560f0eeb30e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 -- 172.21.15.90:0/2478927001 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f038c028040 con 0x560f0eea3780 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f03a0b7d700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f038c037040 tx=0x7f038c00b8e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7399c5e71aba2898 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.757+0000 7f037bfff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f038c02f420 con 0x560f0eea3780 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f038c02f580 con 0x560f0eea3780 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0394084040 con 0x560f0eea3780 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f038c02f740 con 0x560f0eea3780 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f03aef95b80 10 client.16005 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:49.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.758+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560f0ec7acd0 con 0x560f0eea3780 2024-04-06T14:08:49.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0394084040 con 0x560f0eea3780 2024-04-06T14:08:49.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f038c03e780 con 0x560f0eea3780 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 10 client.16005 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 10 client.16005 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 20 client.16005 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0ef12820 0x560f0ef14c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 4 client.16005 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.759+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x560f0ec43cf0 con 0x560f0ef12820 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.760+0000 7f039bfff700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0ef12820 0x560f0ef14c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.760+0000 7f037bfff700 10 client.16005 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.760+0000 7f039bfff700 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0ef12820 0x560f0ef14c40 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=89f401d2bd799765 server_cookie=982c5206aacbb2a9 in_seq=0 out_seq=0 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.761+0000 7f037bfff700 1 -- 172.21.15.90:0/2478927001 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x560f0ec43cf0 con 0x560f0ef12820 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.761+0000 7f037bfff700 10 client.16005 handle_command_reply: tid=0 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.761+0000 7f03aef95b80 1 client.16005 shutdown 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 msgr2=0x560f0eeb4ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 secure :-1 s=READY pgs=520 cs=0 l=1 rev1=1 crypto rx=0x7f038c037040 tx=0x7f038c00b8e0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 shutdown_connections 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560f0ef12820 0x560f0ef14c40 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560f0eeadde0 0x560f0eeb3820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560f0eea3780 0x560f0eeb4ca0 unknown :-1 s=CLOSED pgs=520 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 --2- 172.21.15.90:0/2478927001 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560f0eea40a0 0x560f0eeb30e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 >> 172.21.15.90:0/2478927001 conn(0x560f0ee026a0 msgr2=0x560f0eea2770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f037bfff700 0 client.16005 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 shutdown_connections 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.762+0000 7f03aef95b80 1 -- 172.21.15.90:0/2478927001 wait complete. 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.763+0000 7f03aef95b80 20 client.16005 trim_cache size 0 max 16384 2024-04-06T14:08:49.765 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:49.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 msgr2=0x7f03840a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 0x7f03840a3750 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f038c009990 tx=0x7f038c019040 comp rx=0 tx=0).stop 2024-04-06T14:08:49.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 msgr2=0x7f039c19da00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f039400ef30 tx=0x7f039400c3a0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 shutdown_connections 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f03840a1290 0x7f03840a3750 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f039c112d00 0x7f039c19da00 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f039c109750 0x7f039c1a5590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 --2- 172.21.15.90:0/4280518194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f039c10a070 0x7f039c1a5ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.764+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 >> 172.21.15.90:0/4280518194 conn(0x7f039c06c250 msgr2=0x7f039c1132e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.765+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 shutdown_connections 2024-04-06T14:08:49.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.766+0000 7f03a2de1700 1 -- 172.21.15.90:0/4280518194 wait complete. 2024-04-06T14:08:49.785 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:08:49.785 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:08:49.834 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:49 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3737011409' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:49.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:49 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3737011409' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:49.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.985+0000 7f27f0e11700 1 -- 172.21.15.90:0/2299249343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec105da0 msgr2=0x7f27ec106180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.985+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2299249343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec105da0 0x7f27ec106180 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f27dc009990 tx=0x7f27dc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 -- 172.21.15.90:0/2299249343 shutdown_connections 2024-04-06T14:08:49.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2299249343 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec1008c0 0x7f27ec10cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2299249343 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec105da0 0x7f27ec106180 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2299249343 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec0fff00 0x7f27ec100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 -- 172.21.15.90:0/2299249343 >> 172.21.15.90:0/2299249343 conn(0x7f27ec0769b0 msgr2=0x7f27ec076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:49.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 -- 172.21.15.90:0/2299249343 shutdown_connections 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 -- 172.21.15.90:0/2299249343 wait complete. 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.986+0000 7f27f0e11700 1 Processor -- start 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 -- start start 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 0x7f27ec0723f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec1008c0 0x7f27ec072930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 0x7f27ec06b390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27ec06baa0 con 0x7f27ec105da0 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27ec06bc10 con 0x7f27ec1008c0 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27f0e11700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f27ec06bdd0 con 0x7f27ec0fff00 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27ead9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 0x7f27ec06b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27ead9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 0x7f27ec06b390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59474/0 (socket says 172.21.15.90:59474) 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27ead9d700 1 -- 172.21.15.90:0/2342048963 learned_addr learned my addr 172.21.15.90:0/2342048963 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27ea59c700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 0x7f27ec0723f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.987+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec1008c0 0x7f27ec072930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 -- 172.21.15.90:0/2342048963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec1008c0 msgr2=0x7f27ec072930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec1008c0 0x7f27ec072930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 -- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 msgr2=0x7f27ec06b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 0x7f27ec06b390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 -- 172.21.15.90:0/2342048963 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27dc0095f0 con 0x7f27ec0fff00 2024-04-06T14:08:49.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27ea59c700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 0x7f27ec0723f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f27dc009990 tx=0x7f27dc003a60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.988+0000 7f27db7fe700 1 -- 172.21.15.90:0/2342048963 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27dc01d070 con 0x7f27ec0fff00 2024-04-06T14:08:49.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.989+0000 7f27db7fe700 1 -- 172.21.15.90:0/2342048963 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f27dc003e60 con 0x7f27ec0fff00 2024-04-06T14:08:49.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.989+0000 7f27db7fe700 1 -- 172.21.15.90:0/2342048963 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27dc00fb60 con 0x7f27ec0fff00 2024-04-06T14:08:49.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.990+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f27ec06c050 con 0x7f27ec0fff00 2024-04-06T14:08:49.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.990+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27ec1a6490 con 0x7f27ec0fff00 2024-04-06T14:08:49.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.993+0000 7f27db7fe700 1 -- 172.21.15.90:0/2342048963 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f27dc00fcc0 con 0x7f27ec0fff00 2024-04-06T14:08:49.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.994+0000 7f27db7fe700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 0x7f27d40a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.994+0000 7f27db7fe700 1 -- 172.21.15.90:0/2342048963 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f27dc0cde90 con 0x7f27ec0fff00 2024-04-06T14:08:49.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.994+0000 7f27fcfc5b80 1 Processor -- start 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 -- start start 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a836e780 0x55b6a836eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a837b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a82c6990 con 0x55b6a8378de0 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a82c4350 con 0x55b6a836f0a0 2024-04-06T14:08:49.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a82bfc40 con 0x55b6a836e780 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42202/0 (socket says 172.21.15.90:42202) 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3162866508 learned_addr learned my addr 172.21.15.90:0/3162866508 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27ead9d700 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a837b1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.995+0000 7f27ea59c700 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a836e780 0x55b6a836eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3162866508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a836e780 msgr2=0x55b6a836eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a836e780 0x55b6a836eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3162866508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 msgr2=0x55b6a837b1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a837b1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3162866508 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b6a81413b0 con 0x55b6a836f0a0 2024-04-06T14:08:49.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f27e000be30 tx=0x7f27e00126d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=66871e9947b8d4a3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:49.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:49 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3737011409' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 0x7f27d40a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27d8ff9700 1 -- 172.21.15.90:0/3162866508 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e0010450 con 0x55b6a836f0a0 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27d8ff9700 1 -- 172.21.15.90:0/3162866508 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f27e0010a70 con 0x55b6a836f0a0 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.996+0000 7f27d8ff9700 1 -- 172.21.15.90:0/3162866508 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e001ab70 con 0x55b6a836f0a0 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3162866508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 msgr2=0x55b6a83788a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 0x7f27d40a3850 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f27e001afd0 tx=0x7f27e000f9f0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f27e000be30 tx=0x7f27e00126d0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3162866508 shutdown_connections 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a837b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a836e780 0x55b6a836eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3162866508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a83788a0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3162866508 >> 172.21.15.90:0/3162866508 conn(0x55b6a82cd6a0 msgr2=0x55b6a8369a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.997+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3162866508 shutdown_connections 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3162866508 wait complete. 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 -- start start 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 Processor -- start 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a8378870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8378de0 0x55b6a83762a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a825bd80 con 0x55b6a836e780 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a825ba20 con 0x55b6a836f0a0 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a81436a0 con 0x55b6a8378de0 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.998+0000 7f27fcfc5b80 10 client.0 _ll_register_callbacks cb 0x55b6a8265470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42214/0 (socket says 172.21.15.90:42214) 2024-04-06T14:08:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3227968153 learned_addr learned my addr 172.21.15.90:0/3227968153 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27ea59c700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a8378870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3227968153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8378de0 msgr2=0x55b6a83762a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8378de0 0x55b6a83762a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3227968153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 msgr2=0x55b6a8378870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:49.999+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a8378870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.000+0000 7f27e9d9b700 1 -- 172.21.15.90:0/3227968153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27e000ba90 con 0x55b6a836f0a0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.001+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f27e000b430 tx=0x7f27e0017000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d59cca9c7b041257 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.001+0000 7f27d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.001+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e0032070 con 0x55b6a836f0a0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.001+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f27e002a040 con 0x55b6a836f0a0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.001+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b6a8280380 con 0x55b6a836f0a0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.002+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f27e0012c30 con 0x55b6a836f0a0 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.002+0000 7f27fcfc5b80 10 client.25700 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.002+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b6a80edb60 con 0x55b6a836f0a0 2024-04-06T14:08:50.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.002+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b6a8280380 con 0x55b6a836f0a0 2024-04-06T14:08:50.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f27e0035020 con 0x55b6a836f0a0 2024-04-06T14:08:50.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 10 client.25700 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:50.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 10 client.25700 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 20 client.25700 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83889b0 0x55b6a838add0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 4 client.25700 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.003+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b6a82b8700 con 0x55b6a83889b0 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.004+0000 7f27ea59c700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83889b0 0x55b6a838add0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.004+0000 7f27d1ffb700 10 client.25700 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:50.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.004+0000 7f27ea59c700 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83889b0 0x55b6a838add0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6a7e3401dd19c40f server_cookie=6f9e9b3c6d4a8fba in_seq=0 out_seq=0 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.006+0000 7f27d1ffb700 1 -- 172.21.15.90:0/3227968153 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b6a82b8700 con 0x55b6a83889b0 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.006+0000 7f27d1ffb700 10 client.25700 handle_command_reply: tid=0 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.006+0000 7f27fcfc5b80 1 client.25700 shutdown 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.006+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 msgr2=0x55b6a8375b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.006+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f27e000b430 tx=0x7f27e0017000 comp rx=0 tx=0).stop 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 shutdown_connections 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83889b0 0x55b6a838add0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8378de0 0x55b6a83762a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a8378870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27d1ffb700 0 client.25700 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/3227968153 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836f0a0 0x55b6a8375b60 secure :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f27e000b430 tx=0x7f27e0017000 comp rx=0 tx=0).stop 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.008+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 >> 172.21.15.90:0/3227968153 conn(0x55b6a82cd6a0 msgr2=0x55b6a8368c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.009+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 shutdown_connections 2024-04-06T14:08:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.009+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/3227968153 wait complete. 2024-04-06T14:08:50.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.009+0000 7f27fcfc5b80 20 client.25700 trim_cache size 0 max 16384 2024-04-06T14:08:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 Processor -- start 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 -- start start 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836e780 0x55b6a837fa60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a8369520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a8369a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a8376880 con 0x55b6a8378de0 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a837ffa0 con 0x55b6a836e780 2024-04-06T14:08:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.038+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a8369fa0 con 0x55b6a8365890 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.041+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a8369520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.041+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a8369520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59502/0 (socket says 172.21.15.90:59502) 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.041+0000 7f27e9d9b700 1 -- 172.21.15.90:0/1657128671 learned_addr learned my addr 172.21.15.90:0/1657128671 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.042+0000 7f27ead9d700 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a8369a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.043+0000 7f27ead9d700 1 -- 172.21.15.90:0/1657128671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836e780 msgr2=0x55b6a837fa60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.043+0000 7f27ead9d700 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836e780 0x55b6a837fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.043+0000 7f27ead9d700 1 -- 172.21.15.90:0/1657128671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 msgr2=0x55b6a8369520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.043+0000 7f27ead9d700 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a8369520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.043+0000 7f27ead9d700 1 -- 172.21.15.90:0/1657128671 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b6a8285ea0 con 0x55b6a8365890 2024-04-06T14:08:50.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.048+0000 7f27ead9d700 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a8369a60 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f27e4007470 tx=0x7f27e4009710 comp rx=0 tx=0).ready entity=mon.1 client_cookie=47fa4eb8e4a4bce8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:50.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.051+0000 7f27d97fa700 1 -- 172.21.15.90:0/1657128671 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e400e030 con 0x55b6a8365890 2024-04-06T14:08:50.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.051+0000 7f27d97fa700 1 -- 172.21.15.90:0/1657128671 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f27e4007d40 con 0x55b6a8365890 2024-04-06T14:08:50.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.051+0000 7f27d97fa700 1 -- 172.21.15.90:0/1657128671 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e40145f0 con 0x55b6a8365890 2024-04-06T14:08:50.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.053+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1657128671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 msgr2=0x55b6a8369a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.053+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a8369a60 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f27e4007470 tx=0x7f27e4009710 comp rx=0 tx=0).stop 2024-04-06T14:08:50.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.054+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1657128671 shutdown_connections 2024-04-06T14:08:50.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.054+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a8369a60 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.054+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a836e780 0x55b6a837fa60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.054+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1657128671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a8378de0 0x55b6a8369520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.054+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1657128671 >> 172.21.15.90:0/1657128671 conn(0x55b6a82cd6a0 msgr2=0x55b6a8374450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.066+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1657128671 shutdown_connections 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.068+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1657128671 wait complete. 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.068+0000 7f27fcfc5b80 1 -- start start 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.068+0000 7f27fcfc5b80 1 Processor -- start 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a837f8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a8378de0 0x55b6a837e3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a837ea90 con 0x55b6a836e780 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a837ec00 con 0x55b6a8378de0 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6a83dc830 con 0x55b6a8365890 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27fcfc5b80 10 client.0 _ll_register_callbacks cb 0x55b6a8265470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27e9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59504/0 (socket says 172.21.15.90:59504) 2024-04-06T14:08:50.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.069+0000 7f27e9d9b700 1 -- 172.21.15.90:0/1795189574 learned_addr learned my addr 172.21.15.90:0/1795189574 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:50.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.073+0000 7f27ea59c700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a837f8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27ead9d700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a8378de0 0x55b6a837e3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 -- 172.21.15.90:0/1795189574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 msgr2=0x55b6a837f8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a837f8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 -- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a8378de0 msgr2=0x55b6a837e3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a8378de0 0x55b6a837e3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 -- 172.21.15.90:0/1795189574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27e4009d60 con 0x55b6a836e780 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27e9d9b700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f27e00058f0 tx=0x7f27e000b480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9cf2f503319e0a0c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:50.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.074+0000 7f27d1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:50.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.075+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f27e0032040 con 0x55b6a836e780 2024-04-06T14:08:50.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.075+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f27e0010450 con 0x55b6a836e780 2024-04-06T14:08:50.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.075+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f27e0010a70 con 0x55b6a836e780 2024-04-06T14:08:50.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.075+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f27e0010610 con 0x55b6a836e780 2024-04-06T14:08:50.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 10 client.16023 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:50.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b6a8145cd0 con 0x55b6a836e780 2024-04-06T14:08:50.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f27e0010a70 con 0x55b6a836e780 2024-04-06T14:08:50.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f27e0012740 con 0x55b6a836e780 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 10 client.16023 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 10 client.16023 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 20 client.16023 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83ddb10 0x55b6a83dff30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 4 client.16023 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.077+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b6a810ecf0 con 0x55b6a83ddb10 2024-04-06T14:08:50.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.078+0000 7f27ea59c700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83ddb10 0x55b6a83dff30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:50.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27ea59c700 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83ddb10 0x55b6a83dff30 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ff9c150941e5c993 server_cookie=1c200c52879225d9 in_seq=0 out_seq=0 2024-04-06T14:08:50.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27d1ffb700 10 client.16023 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:50.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27d1ffb700 1 -- 172.21.15.90:0/1795189574 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b6a810ecf0 con 0x55b6a83ddb10 2024-04-06T14:08:50.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27d1ffb700 10 client.16023 handle_command_reply: tid=0 2024-04-06T14:08:50.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27fcfc5b80 1 client.16023 shutdown 2024-04-06T14:08:50.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 msgr2=0x55b6a837de70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.079+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 secure :-1 s=READY pgs=521 cs=0 l=1 rev1=1 crypto rx=0x7f27e00058f0 tx=0x7f27e000b480 comp rx=0 tx=0).stop 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 shutdown_connections 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b6a8378de0 0x55b6a837e3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b6a83ddb10 0x55b6a83dff30 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b6a8365890 0x55b6a837f8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 --2- 172.21.15.90:0/1795189574 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b6a836e780 0x55b6a837de70 unknown :-1 s=CLOSED pgs=521 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 >> 172.21.15.90:0/1795189574 conn(0x55b6a82cd6a0 msgr2=0x55b6a8374380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27d1ffb700 0 client.16023 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 shutdown_connections 2024-04-06T14:08:50.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.082+0000 7f27fcfc5b80 1 -- 172.21.15.90:0/1795189574 wait complete. 2024-04-06T14:08:50.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.083+0000 7f27fcfc5b80 20 client.16023 trim_cache size 0 max 16384 2024-04-06T14:08:50.084 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 msgr2=0x7f27d40a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 0x7f27d40a3850 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f27e001afd0 tx=0x7f27e000f9f0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 msgr2=0x7f27ec0723f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 0x7f27ec0723f0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f27dc009990 tx=0x7f27dc003a60 comp rx=0 tx=0).stop 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 shutdown_connections 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27d40a1390 0x7f27d40a3850 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f27ec105da0 0x7f27ec06b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f27ec0fff00 0x7f27ec0723f0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 --2- 172.21.15.90:0/2342048963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27ec1008c0 0x7f27ec072930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:50.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 >> 172.21.15.90:0/2342048963 conn(0x7f27ec0769b0 msgr2=0x7f27ec0fec70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:50.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 shutdown_connections 2024-04-06T14:08:50.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:50.084+0000 7f27f0e11700 1 -- 172.21.15.90:0/2342048963 wait complete. 2024-04-06T14:08:50.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:50 smithi105 ceph-mon[46969]: pgmap v106: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 94 KiB/s rd, 43 MiB/s wr, 2.17k op/s 2024-04-06T14:08:51.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:50 smithi090 ceph-mon[43420]: pgmap v106: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 94 KiB/s rd, 43 MiB/s wr, 2.17k op/s 2024-04-06T14:08:51.103 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:08:51.103 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:08:51.103 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:08:51.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:50 smithi134 ceph-mon[47549]: pgmap v106: 97 pgs: 97 active+clean; 5.1 GiB data, 7.0 GiB used, 1.0 TiB / 1.0 TiB avail; 94 KiB/s rd, 43 MiB/s wr, 2.17k op/s 2024-04-06T14:08:51.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.309+0000 7fe6b02d1700 1 -- 172.21.15.90:0/3629921172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a811a590 msgr2=0x7fe6a811c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.309+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/3629921172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a811a590 0x7fe6a811c980 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fe6a001c0d0 tx=0x7fe6a001c3e0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.310+0000 7fe6b02d1700 1 -- 172.21.15.90:0/3629921172 shutdown_connections 2024-04-06T14:08:51.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.310+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/3629921172 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a811a590 0x7fe6a811c980 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.310+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/3629921172 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe6a810f4b0 0x7fe6a8107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.310+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/3629921172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a810f920 0x7fe6a81082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.310+0000 7fe6b02d1700 1 -- 172.21.15.90:0/3629921172 >> 172.21.15.90:0/3629921172 conn(0x7fe6a806be00 msgr2=0x7fe6a806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.312+0000 7fe6b02d1700 1 -- 172.21.15.90:0/3629921172 shutdown_connections 2024-04-06T14:08:51.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.312+0000 7fe6b02d1700 1 -- 172.21.15.90:0/3629921172 wait complete. 2024-04-06T14:08:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.312+0000 7fe6b02d1700 1 Processor -- start 2024-04-06T14:08:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 -- start start 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe6a810f4b0 0x7fe6a8115cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a810f920 0x7fe6a8116230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6a8119bb0 con 0x7fe6a810f4b0 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6a8119d20 con 0x7fe6a811a590 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6b02d1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe6a8119ee0 con 0x7fe6a810f920 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42250/0 (socket says 172.21.15.90:42250) 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 -- 172.21.15.90:0/1801470311 learned_addr learned my addr 172.21.15.90:0/1801470311 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a810f920 0x7fe6a8116230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 -- 172.21.15.90:0/1801470311 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a810f920 msgr2=0x7fe6a8116230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a810f920 0x7fe6a8116230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 -- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe6a810f4b0 msgr2=0x7fe6a8115cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe6a810f4b0 0x7fe6a8115cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 -- 172.21.15.90:0/1801470311 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe6a001ad10 con 0x7fe6a811a590 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.313+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fe6a001c7b0 tx=0x7fe6a001a590 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.314+0000 7fe69f7fe700 1 -- 172.21.15.90:0/1801470311 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a001f040 con 0x7fe6a811a590 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.314+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe6a811a160 con 0x7fe6a811a590 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.314+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6a80731c0 con 0x7fe6a811a590 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.315+0000 7fe69f7fe700 1 -- 172.21.15.90:0/1801470311 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe6a00187d0 con 0x7fe6a811a590 2024-04-06T14:08:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.316+0000 7fe69f7fe700 1 -- 172.21.15.90:0/1801470311 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a00052a0 con 0x7fe6a811a590 2024-04-06T14:08:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.316+0000 7fe69f7fe700 1 -- 172.21.15.90:0/1801470311 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fe6a0018260 con 0x7fe6a811a590 2024-04-06T14:08:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.317+0000 7fe69f7fe700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 0x7fe6940a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.317+0000 7fe69f7fe700 1 -- 172.21.15.90:0/1801470311 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe6a000d030 con 0x7fe6a811a590 2024-04-06T14:08:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 Processor -- start 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 -- start start 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82a9800 0x5651e82a9be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b6250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8201a10 con 0x5651e82a9800 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e81ff3d0 con 0x5651e82aa120 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e81facc0 con 0x5651e82b3e60 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6ae86e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b6250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6ae86e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b6250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38116/0 (socket says 172.21.15.90:38116) 2024-04-06T14:08:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42264/0 (socket says 172.21.15.90:42264) 2024-04-06T14:08:51.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.320+0000 7fe6ae86e700 1 -- 172.21.15.90:0/2693838024 learned_addr learned my addr 172.21.15.90:0/2693838024 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.321+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2693838024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 msgr2=0x5651e82b6250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.321+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b6250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.321+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2693838024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82a9800 msgr2=0x5651e82a9be0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:08:51.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.321+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82a9800 0x5651e82a9be0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.321+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2693838024 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5651e807c430 con 0x5651e82aa120 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.322+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7fe6a810fd40 tx=0x7fe6a4009f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e07385f27ef387f7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.322+0000 7fe69cda5700 1 -- 172.21.15.90:0/2693838024 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4011c80 con 0x5651e82aa120 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.322+0000 7fe69cda5700 1 -- 172.21.15.90:0/2693838024 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe6a4011de0 con 0x5651e82aa120 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.322+0000 7fe69cda5700 1 -- 172.21.15.90:0/2693838024 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4005280 con 0x5651e82aa120 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.323+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 0x7fe6940a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.323+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2693838024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 msgr2=0x5651e82b3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.323+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7fe6a810fd40 tx=0x7fe6a4009f60 comp rx=0 tx=0).stop 2024-04-06T14:08:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.323+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 0x7fe6940a3910 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fe6980096b0 tx=0x7fe698006c70 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.326+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2693838024 shutdown_connections 2024-04-06T14:08:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.326+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.326+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82a9800 0x5651e82a9be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.326+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2693838024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82aa120 0x5651e82b3920 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.326+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2693838024 >> 172.21.15.90:0/2693838024 conn(0x5651e8208720 msgr2=0x5651e82a4b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.327+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2693838024 shutdown_connections 2024-04-06T14:08:51.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.327+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2693838024 wait complete. 2024-04-06T14:08:51.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 -- start start 2024-04-06T14:08:51.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 Processor -- start 2024-04-06T14:08:51.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b38a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b1370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8196e00 con 0x5651e82aa120 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8196aa0 con 0x5651e82a9800 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e807e720 con 0x5651e82b3e60 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6bc485b80 10 client.0 _ll_register_callbacks cb 0x5651e81a04f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59512/0 (socket says 172.21.15.90:59512) 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6ad86c700 1 -- 172.21.15.90:0/1046419701 learned_addr learned my addr 172.21.15.90:0/1046419701 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.328+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b1370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 -- 172.21.15.90:0/1046419701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 msgr2=0x5651e82b1370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b1370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 -- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 msgr2=0x5651e82b38a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b38a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 -- 172.21.15.90:0/1046419701 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe6a40098e0 con 0x5651e82aa120 2024-04-06T14:08:51.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7fe6a4004d10 tx=0x7fe6a4005ce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2f4b241b1f32f49d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe691ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4004050 con 0x5651e82aa120 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe6a40041b0 con 0x5651e82aa120 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.329+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5651e81bb400 con 0x5651e82aa120 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe6a4004370 con 0x5651e82aa120 2024-04-06T14:08:51.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 10 client.16026 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5651e8028be0 con 0x5651e82aa120 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5651e81bb400 con 0x5651e82aa120 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe6a40052f0 con 0x5651e82aa120 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 10 client.16026 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 10 client.16026 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 20 client.16026 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e82c3940 0x5651e82c5d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 4 client.16026 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.330+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5651e81f3780 con 0x5651e82c3940 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.331+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e82c3940 0x5651e82c5d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.332+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e82c3940 0x5651e82c5d60 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6050867b0c5748e server_cookie=e7f48a422f737d24 in_seq=0 out_seq=0 2024-04-06T14:08:51.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.332+0000 7fe691ffb700 10 client.16026 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.333+0000 7fe691ffb700 1 -- 172.21.15.90:0/1046419701 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5651e81f3780 con 0x5651e82c3940 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.333+0000 7fe691ffb700 10 client.16026 handle_command_reply: tid=0 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.333+0000 7fe6bc485b80 1 client.16026 shutdown 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.333+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 msgr2=0x5651e82b0c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.333+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 secure :-1 s=READY pgs=522 cs=0 l=1 rev1=1 crypto rx=0x7fe6a4004d10 tx=0x7fe6a4005ce0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 shutdown_connections 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e82c3940 0x5651e82c5d60 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e82b1370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b38a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe691ffb700 0 client.16026 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/1046419701 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b0c30 unknown :-1 s=CLOSED pgs=522 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 >> 172.21.15.90:0/1046419701 conn(0x5651e8208720 msgr2=0x5651e82a3ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 shutdown_connections 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 1 -- 172.21.15.90:0/1046419701 wait complete. 2024-04-06T14:08:51.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.334+0000 7fe6bc485b80 20 client.16026 trim_cache size 0 max 16384 2024-04-06T14:08:51.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 Processor -- start 2024-04-06T14:08:51.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 -- start start 2024-04-06T14:08:51.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82a9800 0x5651e82b0550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82b3e60 0x5651e82a8f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e829c430 con 0x5651e82aa120 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e82bb0b0 con 0x5651e82b3e60 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e82ac850 con 0x5651e82a9800 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6ad86c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59516/0 (socket says 172.21.15.90:59516) 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2035311468 learned_addr learned my addr 172.21.15.90:0/2035311468 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82a9800 0x5651e82b0550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.358+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82b3e60 0x5651e82a8f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2035311468 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82a9800 msgr2=0x5651e82b0550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82a9800 0x5651e82b0550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2035311468 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82b3e60 msgr2=0x5651e82a8f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82b3e60 0x5651e82a8f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 -- 172.21.15.90:0/2035311468 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5651e82a0be0 con 0x5651e82aa120 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7fe6a4004b00 tx=0x7fe6a4005e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e277061ade8b0d97 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe69d5a6700 1 -- 172.21.15.90:0/2035311468 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a402a3a0 con 0x5651e82aa120 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe69d5a6700 1 -- 172.21.15.90:0/2035311468 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe6a40224a0 con 0x5651e82aa120 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.359+0000 7fe69d5a6700 1 -- 172.21.15.90:0/2035311468 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a4022bb0 con 0x5651e82aa120 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.360+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2035311468 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 msgr2=0x5651e82bab40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.360+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 secure :-1 s=READY pgs=523 cs=0 l=1 rev1=1 crypto rx=0x7fe6a4004b00 tx=0x7fe6a4005e50 comp rx=0 tx=0).stop 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2035311468 shutdown_connections 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82b3e60 0x5651e82a8f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82a9800 0x5651e82b0550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/2035311468 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82bab40 unknown :-1 s=CLOSED pgs=523 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2035311468 >> 172.21.15.90:0/2035311468 conn(0x5651e8208720 msgr2=0x5651e82a87f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2035311468 shutdown_connections 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 -- 172.21.15.90:0/2035311468 wait complete. 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.363+0000 7fe6bc485b80 1 -- start start 2024-04-06T14:08:51.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 Processor -- start 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b99f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b9f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e83118a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8311de0 con 0x5651e82aa120 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8311f50 con 0x5651e82a9800 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651e8312110 con 0x5651e82b3e60 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6bc485b80 10 client.0 _ll_register_callbacks cb 0x5651e81a04f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e83118a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae06d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b99f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae06d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b99f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42296/0 (socket says 172.21.15.90:42296) 2024-04-06T14:08:51.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae06d700 1 -- 172.21.15.90:0/752254931 learned_addr learned my addr 172.21.15.90:0/752254931 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ad86c700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b9f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 -- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 msgr2=0x5651e82b99f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b99f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 -- 172.21.15.90:0/752254931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 msgr2=0x5651e82b9f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b9f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.364+0000 7fe6ae86e700 1 -- 172.21.15.90:0/752254931 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5651e8028be0 con 0x5651e82b3e60 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe6ae86e700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e83118a0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fe6a0018fa0 tx=0x7fe6a001dfd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6c7eb3d9b2aa00e7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe691ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe6a00c1bc0 con 0x5651e82b3e60 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fe6a00c6180 con 0x5651e82b3e60 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe6a4011de0 con 0x5651e82b3e60 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe6a00be200 con 0x5651e82b3e60 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe6bc485b80 10 client.25633 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:51.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.365+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5651e81c0f20 con 0x5651e82b3e60 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe6a4011de0 con 0x5651e82b3e60 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe6a00c5cf0 con 0x5651e82b3e60 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 10 client.25633 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 10 client.25633 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 20 client.25633 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e8319790 0x5651e831bbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 4 client.25633 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x5651e8049d70 con 0x5651e8319790 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.366+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e8319790 0x5651e831bbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.367+0000 7fe691ffb700 10 client.25633 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.367+0000 7fe6ae06d700 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e8319790 0x5651e831bbb0 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1d40606565f02d7f server_cookie=c3920123beccd7db in_seq=0 out_seq=0 2024-04-06T14:08:51.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe691ffb700 1 -- 172.21.15.90:0/752254931 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x5651e8049d70 con 0x5651e8319790 2024-04-06T14:08:51.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe691ffb700 10 client.25633 handle_command_reply: tid=0 2024-04-06T14:08:51.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 client.25633 shutdown 2024-04-06T14:08:51.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 msgr2=0x5651e83118a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e83118a0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fe6a0018fa0 tx=0x7fe6a001dfd0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 shutdown_connections 2024-04-06T14:08:51.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5651e8319790 0x5651e831bbb0 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651e82b3e60 0x5651e83118a0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651e82a9800 0x5651e82b99f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 --2- 172.21.15.90:0/752254931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651e82aa120 0x5651e82b9f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 >> 172.21.15.90:0/752254931 conn(0x5651e8208720 msgr2=0x5651e82a9080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.371+0000 7fe691ffb700 0 client.25633 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.372+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 shutdown_connections 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.372+0000 7fe6bc485b80 1 -- 172.21.15.90:0/752254931 wait complete. 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.372+0000 7fe6bc485b80 20 client.25633 trim_cache size 0 max 16384 2024-04-06T14:08:51.373 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:51.374 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:08:51.374 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "a30de293-7114-4ea3-b39f-c08e0561198e", 2024-04-06T14:08:51.374 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:08:51.374 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 msgr2=0x7fe6940a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 0x7fe6940a3910 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fe6980096b0 tx=0x7fe698006c70 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 msgr2=0x7fe6a8119570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fe6a001c7b0 tx=0x7fe6a001a590 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 shutdown_connections 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe6940a1450 0x7fe6940a3910 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe6a811a590 0x7fe6a8119570 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe6a810f4b0 0x7fe6a8115cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 --2- 172.21.15.90:0/1801470311 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe6a810f920 0x7fe6a8116230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.373+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 >> 172.21.15.90:0/1801470311 conn(0x7fe6a806be00 msgr2=0x7fe6a811a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.374+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 shutdown_connections 2024-04-06T14:08:51.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.374+0000 7fe6b02d1700 1 -- 172.21.15.90:0/1801470311 wait complete. 2024-04-06T14:08:51.394 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:08:51.394 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:08:51.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 -- 172.21.15.90:0/2824330901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c11a590 msgr2=0x7fec1c11c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 --2- 172.21.15.90:0/2824330901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c11a590 0x7fec1c11c980 secure :-1 s=READY pgs=524 cs=0 l=1 rev1=1 crypto rx=0x7fec1401c0d0 tx=0x7fec1401c3e0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 -- 172.21.15.90:0/2824330901 shutdown_connections 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 --2- 172.21.15.90:0/2824330901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c11a590 0x7fec1c11c980 unknown :-1 s=CLOSED pgs=524 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 --2- 172.21.15.90:0/2824330901 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec1c10f4b0 0x7fec1c107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 --2- 172.21.15.90:0/2824330901 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec1c10f920 0x7fec1c1082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 -- 172.21.15.90:0/2824330901 >> 172.21.15.90:0/2824330901 conn(0x7fec1c06be00 msgr2=0x7fec1c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 -- 172.21.15.90:0/2824330901 shutdown_connections 2024-04-06T14:08:51.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.595+0000 7fec20f9e700 1 -- 172.21.15.90:0/2824330901 wait complete. 2024-04-06T14:08:51.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.596+0000 7fec20f9e700 1 Processor -- start 2024-04-06T14:08:51.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 -- start start 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec1c10f920 0x7fec1c11a030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec1c11a590 0x7fec1c115d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1c1164f0 con 0x7fec1c10f4b0 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1c116660 con 0x7fec1c10f920 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec20f9e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1c116820 con 0x7fec1c11a590 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec1a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59558/0 (socket says 172.21.15.90:59558) 2024-04-06T14:08:51.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.597+0000 7fec1a59c700 1 -- 172.21.15.90:0/3987917902 learned_addr learned my addr 172.21.15.90:0/3987917902 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 -- 172.21.15.90:0/3987917902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec1c11a590 msgr2=0x7fec1c115d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec1c11a590 0x7fec1c115d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 -- 172.21.15.90:0/3987917902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec1c10f920 msgr2=0x7fec1c11a030 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec1c10f920 0x7fec1c11a030 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 -- 172.21.15.90:0/3987917902 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec1401ad10 con 0x7fec1c10f4b0 2024-04-06T14:08:51.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.598+0000 7fec1a59c700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fec0c00d950 tx=0x7fec0c00dd10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.599+0000 7fec0b7fe700 1 -- 172.21.15.90:0/3987917902 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec0c009ac0 con 0x7fec1c10f4b0 2024-04-06T14:08:51.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.599+0000 7fec0b7fe700 1 -- 172.21.15.90:0/3987917902 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec0c009c20 con 0x7fec1c10f4b0 2024-04-06T14:08:51.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.599+0000 7fec0b7fe700 1 -- 172.21.15.90:0/3987917902 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec0c00f620 con 0x7fec1c10f4b0 2024-04-06T14:08:51.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.600+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec1c1aa220 con 0x7fec1c10f4b0 2024-04-06T14:08:51.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.600+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec1c1aa650 con 0x7fec1c10f4b0 2024-04-06T14:08:51.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.603+0000 7fec0b7fe700 1 -- 172.21.15.90:0/3987917902 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fec0c010460 con 0x7fec1c10f4b0 2024-04-06T14:08:51.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.604+0000 7fec0b7fe700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 0x7fec040a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.604+0000 7fec0b7fe700 1 -- 172.21.15.90:0/3987917902 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fec0c0cca90 con 0x7fec1c10f4b0 2024-04-06T14:08:51.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.604+0000 7fec19d9b700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 0x7fec040a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 Processor -- start 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 -- start start 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab872b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab87f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec19d9b700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 0x7fec040a3960 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fec1c117340 tx=0x7fec1000b370 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab7ca990 con 0x5630ab87cde0 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab7c8350 con 0x5630ab8730a0 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec2d152b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab7c3c40 con 0x5630ab872780 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec19d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec19d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42322/0 (socket says 172.21.15.90:42322) 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec19d9b700 1 -- 172.21.15.90:0/1880040423 learned_addr learned my addr 172.21.15.90:0/1880040423 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.605+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab87f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1a59c700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab872b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 -- 172.21.15.90:0/1880040423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 msgr2=0x5630ab872b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab872b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 -- 172.21.15.90:0/1880040423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 msgr2=0x5630ab87c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 -- 172.21.15.90:0/1880040423 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5630ab6453b0 con 0x5630ab87cde0 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec19d9b700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:08:51.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.606+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab87f1d0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fec1c06a110 tx=0x7fec14019ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2ea60d46f4cbdeac server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.608+0000 7fec08ff9700 1 -- 172.21.15.90:0/1880040423 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec1401f040 con 0x5630ab87cde0 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.608+0000 7fec08ff9700 1 -- 172.21.15.90:0/1880040423 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec14003a00 con 0x5630ab87cde0 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.608+0000 7fec08ff9700 1 -- 172.21.15.90:0/1880040423 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec14005110 con 0x5630ab87cde0 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.608+0000 7fec2d152b80 1 -- 172.21.15.90:0/1880040423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 msgr2=0x5630ab87f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.608+0000 7fec2d152b80 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab87f1d0 secure :-1 s=READY pgs=526 cs=0 l=1 rev1=1 crypto rx=0x7fec1c06a110 tx=0x7fec14019ba0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 -- 172.21.15.90:0/1880040423 shutdown_connections 2024-04-06T14:08:51.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab87f1d0 unknown :-1 s=CLOSED pgs=526 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab872b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 --2- 172.21.15.90:0/1880040423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab87c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 -- 172.21.15.90:0/1880040423 >> 172.21.15.90:0/1880040423 conn(0x5630ab7d16a0 msgr2=0x5630ab86da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.609+0000 7fec2d152b80 1 -- 172.21.15.90:0/1880040423 shutdown_connections 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 -- 172.21.15.90:0/1880040423 wait complete. 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 -- start start 2024-04-06T14:08:51.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 Processor -- start 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab8730a0 0x5630ab879aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab87cde0 0x5630ab87a0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab75fd80 con 0x5630ab8730a0 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab75fa20 con 0x5630ab872780 2024-04-06T14:08:51.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab6476a0 con 0x5630ab87cde0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.610+0000 7fec2d152b80 10 client.0 _ll_register_callbacks cb 0x5630ab769470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42336/0 (socket says 172.21.15.90:42336) 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 -- 172.21.15.90:0/2205400487 learned_addr learned my addr 172.21.15.90:0/2205400487 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab87cde0 0x5630ab87a0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 -- 172.21.15.90:0/2205400487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab87cde0 msgr2=0x5630ab87a0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab87cde0 0x5630ab87a0d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 -- 172.21.15.90:0/2205400487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab8730a0 msgr2=0x5630ab879aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab8730a0 0x5630ab879aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 -- 172.21.15.90:0/2205400487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec1c11d270 con 0x5630ab872780 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fec0c0965e0 tx=0x7fec0c099ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2cf1996c9f599043 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec01ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec0c0ca630 con 0x5630ab872780 2024-04-06T14:08:51.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fec0c0b22a0 con 0x5630ab872780 2024-04-06T14:08:51.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.611+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5630ab784380 con 0x5630ab872780 2024-04-06T14:08:51.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.612+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fec0c0b2460 con 0x5630ab872780 2024-04-06T14:08:51.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.613+0000 7fec2d152b80 10 client.25730 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:51.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.613+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5630ab5f1b60 con 0x5630ab872780 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.613+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5630ab784380 con 0x5630ab872780 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fec0c0990f0 con 0x5630ab872780 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 10 client.25730 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 10 client.25730 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 20 client.25730 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab88c930 0x5630ab88ed50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 4 client.25730 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:08:51.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.614+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5630ab7bc700 con 0x5630ab88c930 2024-04-06T14:08:51.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.615+0000 7fec19d9b700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab88c930 0x5630ab88ed50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.616+0000 7fec19d9b700 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab88c930 0x5630ab88ed50 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1844a4842407b659 server_cookie=3743d507af9f684d in_seq=0 out_seq=0 2024-04-06T14:08:51.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.616+0000 7fec01ffb700 10 client.25730 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec01ffb700 1 -- 172.21.15.90:0/2205400487 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5630ab7bc700 con 0x5630ab88c930 2024-04-06T14:08:51.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec01ffb700 10 client.25730 handle_command_reply: tid=0 2024-04-06T14:08:51.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 client.25730 shutdown 2024-04-06T14:08:51.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 msgr2=0x5630ab87c760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fec0c0965e0 tx=0x7fec0c099ca0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 shutdown_connections 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab88c930 0x5630ab88ed50 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab87cde0 0x5630ab87a0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab87c760 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2205400487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab8730a0 0x5630ab879aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 >> 172.21.15.90:0/2205400487 conn(0x5630ab7d16a0 msgr2=0x5630ab86cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec01ffb700 0 client.25730 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 shutdown_connections 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.617+0000 7fec2d152b80 1 -- 172.21.15.90:0/2205400487 wait complete. 2024-04-06T14:08:51.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.618+0000 7fec2d152b80 20 client.25730 trim_cache size 0 max 16384 2024-04-06T14:08:51.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.641+0000 7fec2d152b80 1 Processor -- start 2024-04-06T14:08:51.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 -- start start 2024-04-06T14:08:51.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab883a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab878370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab883f70 con 0x5630ab87cde0 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab87f1d0 con 0x5630ab8730a0 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec2d152b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab877fb0 con 0x5630ab872780 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec1ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec1ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59592/0 (socket says 172.21.15.90:59592) 2024-04-06T14:08:51.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.642+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2083582441 learned_addr learned my addr 172.21.15.90:0/2083582441 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab883a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2083582441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 msgr2=0x5630ab883a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab883a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2083582441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 msgr2=0x5630ab878370 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:51.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab878370 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2083582441 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5630ab789ea0 con 0x5630ab87cde0 2024-04-06T14:08:51.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fec1c06a110 tx=0x7fec1401ac00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d2f082a58c63dd33 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec097fa700 1 -- 172.21.15.90:0/2083582441 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec1401f070 con 0x5630ab87cde0 2024-04-06T14:08:51.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec097fa700 1 -- 172.21.15.90:0/2083582441 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec14020040 con 0x5630ab87cde0 2024-04-06T14:08:51.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.643+0000 7fec097fa700 1 -- 172.21.15.90:0/2083582441 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec14005370 con 0x5630ab87cde0 2024-04-06T14:08:51.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 -- 172.21.15.90:0/2083582441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 msgr2=0x5630ab8788b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 secure :-1 s=READY pgs=527 cs=0 l=1 rev1=1 crypto rx=0x7fec1c06a110 tx=0x7fec1401ac00 comp rx=0 tx=0).stop 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 -- 172.21.15.90:0/2083582441 shutdown_connections 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab8788b0 unknown :-1 s=CLOSED pgs=527 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab872780 0x5630ab883a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2083582441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab8730a0 0x5630ab878370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.644+0000 7fec2d152b80 1 -- 172.21.15.90:0/2083582441 >> 172.21.15.90:0/2083582441 conn(0x5630ab7d16a0 msgr2=0x5630ab874da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- 172.21.15.90:0/2083582441 shutdown_connections 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- 172.21.15.90:0/2083582441 wait complete. 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- start start 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 Processor -- start 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab883720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab8730a0 0x5630ab881b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab882590 con 0x5630ab87cde0 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab882700 con 0x5630ab872780 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5630ab8828c0 con 0x5630ab8730a0 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec2d152b80 10 client.0 _ll_register_callbacks cb 0x5630ab769470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec1ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec1ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59602/0 (socket says 172.21.15.90:59602) 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.645+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2352923939 learned_addr learned my addr 172.21.15.90:0/2352923939 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab883720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2352923939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab8730a0 msgr2=0x5630ab881b10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab8730a0 0x5630ab881b10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 msgr2=0x5630ab883720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab883720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 -- 172.21.15.90:0/2352923939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5630ab6453b0 con 0x5630ab87cde0 2024-04-06T14:08:51.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec1ad9d700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7fec1401c0d0 tx=0x7fec14003e20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f5a2e9975eac927 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:08:51.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.646+0000 7fec01ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:08:51.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.647+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec1401f040 con 0x5630ab87cde0 2024-04-06T14:08:51.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.647+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fec140184f0 con 0x5630ab87cde0 2024-04-06T14:08:51.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.647+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fec14003a00 con 0x5630ab87cde0 2024-04-06T14:08:51.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fec140186b0 con 0x5630ab87cde0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 10 client.16047 fetch_fsmap learned FSMap version 24 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5630ab649cd0 con 0x5630ab87cde0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec14003a00 con 0x5630ab87cde0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fec1403c070 con 0x5630ab87cde0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 10 client.16047 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 10 client.16047 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 20 client.16047 populate_metadata read hostname 'smithi090' 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab8e17e0 0x5630ab8e3c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 4 client.16047 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.648+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5630ab612cf0 con 0x5630ab8e17e0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab8e17e0 0x5630ab8e3c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec01ffb700 10 client.16047 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec1a59c700 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab8e17e0 0x5630ab8e3c00 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=75cf5e3d2b2c775e server_cookie=7ed2cdff885b5656 in_seq=0 out_seq=0 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec01ffb700 1 -- 172.21.15.90:0/2352923939 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x5630ab612cf0 con 0x5630ab8e17e0 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec01ffb700 10 client.16047 handle_command_reply: tid=0 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.649+0000 7fec2d152b80 1 client.16047 shutdown 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 msgr2=0x5630ab882050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 secure :-1 s=READY pgs=528 cs=0 l=1 rev1=1 crypto rx=0x7fec1401c0d0 tx=0x7fec14003e20 comp rx=0 tx=0).stop 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 shutdown_connections 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5630ab87cde0 0x5630ab882050 unknown :-1 s=CLOSED pgs=528 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5630ab872780 0x5630ab883720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5630ab8e17e0 0x5630ab8e3c00 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 --2- 172.21.15.90:0/2352923939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5630ab8730a0 0x5630ab881b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec01ffb700 0 client.16047 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 >> 172.21.15.90:0/2352923939 conn(0x5630ab7d16a0 msgr2=0x5630ab878310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 shutdown_connections 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 1 -- 172.21.15.90:0/2352923939 wait complete. 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.650+0000 7fec2d152b80 20 client.16047 trim_cache size 0 max 16384 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "a30de293-7114-4ea3-b39f-c08e0561198e": { 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "a30de293-7114-4ea3-b39f-c08e0561198e", 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:08:51.654 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:08:51.655 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:08:51.655 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:08:51.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 msgr2=0x7fec040a3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 0x7fec040a3960 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fec1c117340 tx=0x7fec1000b370 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 msgr2=0x7fec1c119af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 secure :-1 s=READY pgs=525 cs=0 l=1 rev1=1 crypto rx=0x7fec0c00d950 tx=0x7fec0c00dd10 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 shutdown_connections 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec040a14a0 0x7fec040a3960 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec1c11a590 0x7fec1c115d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec1c10f4b0 0x7fec1c119af0 unknown :-1 s=CLOSED pgs=525 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 --2- 172.21.15.90:0/3987917902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec1c10f920 0x7fec1c11a030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:08:51.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.652+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 >> 172.21.15.90:0/3987917902 conn(0x7fec1c06be00 msgr2=0x7fec1c108f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:08:51.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.653+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 shutdown_connections 2024-04-06T14:08:51.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:08:51.653+0000 7fec20f9e700 1 -- 172.21.15.90:0/3987917902 wait complete. 2024-04-06T14:08:51.691 INFO:tasks.cephfs.filesystem:scrub status for tag:a30de293-7114-4ea3-b39f-c08e0561198e - {'path': '/', 'tag': 'a30de293-7114-4ea3-b39f-c08e0561198e', 'options': 'recursive,force'} 2024-04-06T14:08:51.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:51 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:08:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:51 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:51.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:51 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:08:52.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:51 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:08:52.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:51 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:52.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:51 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:08:52.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:51 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:08:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:51 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:52.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:51 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:08:52.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:52 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:52.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:52 smithi105 ceph-mon[46969]: pgmap v107: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 42 MiB/s wr, 2.14k op/s 2024-04-06T14:08:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 185.0 s, 2694.8 tps, lat 37.285 ms stddev 43.037 2024-04-06T14:08:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:52 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:52 smithi090 ceph-mon[43420]: pgmap v107: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 42 MiB/s wr, 2.14k op/s 2024-04-06T14:08:53.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:52 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:08:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:52 smithi134 ceph-mon[47549]: pgmap v107: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 42 MiB/s wr, 2.14k op/s 2024-04-06T14:08:53.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:53 smithi105 ceph-mon[46969]: pgmap v108: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 37 MiB/s wr, 1.90k op/s 2024-04-06T14:08:54.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:53 smithi090 ceph-mon[43420]: pgmap v108: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 37 MiB/s wr, 1.90k op/s 2024-04-06T14:08:54.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:53 smithi134 ceph-mon[47549]: pgmap v108: 97 pgs: 97 active+clean; 5.2 GiB data, 7.1 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 37 MiB/s wr, 1.90k op/s 2024-04-06T14:08:55.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:55 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:08:55.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:55 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:08:56.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:55 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:08:56.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:56 smithi134 ceph-mon[47549]: pgmap v109: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 45 MiB/s wr, 2.24k op/s 2024-04-06T14:08:56.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:56 smithi105 ceph-mon[46969]: pgmap v109: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 45 MiB/s wr, 2.24k op/s 2024-04-06T14:08:57.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:56 smithi090 ceph-mon[43420]: pgmap v109: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 107 KiB/s rd, 45 MiB/s wr, 2.24k op/s 2024-04-06T14:08:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 190.0 s, 2759.2 tps, lat 36.197 ms stddev 36.040 2024-04-06T14:08:58.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:58 smithi134 ceph-mon[47549]: pgmap v110: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 805 op/s 2024-04-06T14:08:58.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:58 smithi105 ceph-mon[46969]: pgmap v110: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 805 op/s 2024-04-06T14:08:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:58 smithi090 ceph-mon[43420]: pgmap v110: 97 pgs: 97 active+clean; 5.2 GiB data, 7.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 805 op/s 2024-04-06T14:08:59.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:08:59 smithi105 ceph-mon[46969]: pgmap v111: 97 pgs: 97 active+clean; 5.3 GiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 MiB/s wr, 962 op/s 2024-04-06T14:09:00.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:08:59 smithi090 ceph-mon[43420]: pgmap v111: 97 pgs: 97 active+clean; 5.3 GiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 MiB/s wr, 962 op/s 2024-04-06T14:09:00.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:08:59 smithi134 ceph-mon[47549]: pgmap v111: 97 pgs: 97 active+clean; 5.3 GiB data, 7.3 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 MiB/s wr, 962 op/s 2024-04-06T14:09:02.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:02 smithi105 ceph-mon[46969]: pgmap v112: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 25 MiB/s wr, 1.01k op/s 2024-04-06T14:09:03.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 195.0 s, 1192.6 tps, lat 83.606 ms stddev 137.032 2024-04-06T14:09:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:02 smithi090 ceph-mon[43420]: pgmap v112: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 25 MiB/s wr, 1.01k op/s 2024-04-06T14:09:03.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:02 smithi134 ceph-mon[47549]: pgmap v112: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 25 MiB/s wr, 1.01k op/s 2024-04-06T14:09:03.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:03 smithi105 ceph-mon[46969]: pgmap v113: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 19 MiB/s wr, 716 op/s 2024-04-06T14:09:04.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:03 smithi090 ceph-mon[43420]: pgmap v113: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 19 MiB/s wr, 716 op/s 2024-04-06T14:09:04.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:03 smithi134 ceph-mon[47549]: pgmap v113: 97 pgs: 97 active+clean; 5.3 GiB data, 7.4 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 19 MiB/s wr, 716 op/s 2024-04-06T14:09:06.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:05 smithi090 ceph-mon[43420]: pgmap v114: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 49 MiB/s wr, 2.30k op/s 2024-04-06T14:09:06.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:05 smithi134 ceph-mon[47549]: pgmap v114: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 49 MiB/s wr, 2.30k op/s 2024-04-06T14:09:06.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:05 smithi105 ceph-mon[46969]: pgmap v114: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 49 MiB/s wr, 2.30k op/s 2024-04-06T14:09:08.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 200.0 s, 2017.0 tps, lat 49.011 ms stddev 53.432 2024-04-06T14:09:08.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:08 smithi134 ceph-mon[47549]: pgmap v115: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 41 MiB/s wr, 1.96k op/s 2024-04-06T14:09:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:08 smithi105 ceph-mon[46969]: pgmap v115: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 41 MiB/s wr, 1.96k op/s 2024-04-06T14:09:09.043 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:08 smithi090 ceph-mon[43420]: pgmap v115: 97 pgs: 97 active+clean; 5.4 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 41 MiB/s wr, 1.96k op/s 2024-04-06T14:09:10.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:09 smithi090 ceph-mon[43420]: pgmap v116: 97 pgs: 97 active+clean; 5.5 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 52 MiB/s wr, 2.70k op/s 2024-04-06T14:09:10.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:09 smithi134 ceph-mon[47549]: pgmap v116: 97 pgs: 97 active+clean; 5.5 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 52 MiB/s wr, 2.70k op/s 2024-04-06T14:09:10.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:09 smithi105 ceph-mon[46969]: pgmap v116: 97 pgs: 97 active+clean; 5.5 GiB data, 7.8 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 52 MiB/s wr, 2.70k op/s 2024-04-06T14:09:12.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:11 smithi090 ceph-mon[43420]: pgmap v117: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 55 MiB/s wr, 2.95k op/s 2024-04-06T14:09:12.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:11 smithi134 ceph-mon[47549]: pgmap v117: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 55 MiB/s wr, 2.95k op/s 2024-04-06T14:09:12.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:11 smithi105 ceph-mon[46969]: pgmap v117: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 55 MiB/s wr, 2.95k op/s 2024-04-06T14:09:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 205.0 s, 2237.4 tps, lat 45.174 ms stddev 64.167 2024-04-06T14:09:14.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:13 smithi090 ceph-mon[43420]: pgmap v118: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 49 MiB/s wr, 2.74k op/s 2024-04-06T14:09:14.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:13 smithi134 ceph-mon[47549]: pgmap v118: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 49 MiB/s wr, 2.74k op/s 2024-04-06T14:09:14.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:13 smithi105 ceph-mon[46969]: pgmap v118: 97 pgs: 97 active+clean; 5.6 GiB data, 7.9 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 49 MiB/s wr, 2.74k op/s 2024-04-06T14:09:16.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:15 smithi090 ceph-mon[43420]: pgmap v119: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 138 KiB/s rd, 60 MiB/s wr, 3.15k op/s 2024-04-06T14:09:16.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:15 smithi134 ceph-mon[47549]: pgmap v119: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 138 KiB/s rd, 60 MiB/s wr, 3.15k op/s 2024-04-06T14:09:16.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:15 smithi105 ceph-mon[46969]: pgmap v119: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 138 KiB/s rd, 60 MiB/s wr, 3.15k op/s 2024-04-06T14:09:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 210.0 s, 2625.6 tps, lat 38.315 ms stddev 41.888 2024-04-06T14:09:18.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:18 smithi134 ceph-mon[47549]: pgmap v120: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 29 MiB/s wr, 1.56k op/s 2024-04-06T14:09:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:18 smithi105 ceph-mon[46969]: pgmap v120: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 29 MiB/s wr, 1.56k op/s 2024-04-06T14:09:19.046 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:18 smithi090 ceph-mon[43420]: pgmap v120: 97 pgs: 97 active+clean; 5.7 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 29 MiB/s wr, 1.56k op/s 2024-04-06T14:09:20.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:19 smithi090 ceph-mon[43420]: pgmap v121: 97 pgs: 97 active+clean; 5.8 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 135 KiB/s rd, 36 MiB/s wr, 1.87k op/s 2024-04-06T14:09:20.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:19 smithi134 ceph-mon[47549]: pgmap v121: 97 pgs: 97 active+clean; 5.8 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 135 KiB/s rd, 36 MiB/s wr, 1.87k op/s 2024-04-06T14:09:20.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:19 smithi105 ceph-mon[46969]: pgmap v121: 97 pgs: 97 active+clean; 5.8 GiB data, 8.2 GiB used, 1.0 TiB / 1.0 TiB avail; 135 KiB/s rd, 36 MiB/s wr, 1.87k op/s 2024-04-06T14:09:21.692 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:09:21.692 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 -- 172.21.15.90:0/1872781951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c1088b0 msgr2=0x7f065c108c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 --2- 172.21.15.90:0/1872781951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c1088b0 0x7f065c108c90 secure :-1 s=READY pgs=529 cs=0 l=1 rev1=1 crypto rx=0x7f064c00da10 tx=0x7f064c00dd20 comp rx=0 tx=0).stop 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 -- 172.21.15.90:0/1872781951 shutdown_connections 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 --2- 172.21.15.90:0/1872781951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 0x7f065c114ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 --2- 172.21.15.90:0/1872781951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c1088b0 0x7f065c108c90 unknown :-1 s=CLOSED pgs=529 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 --2- 172.21.15.90:0/1872781951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c109260 0x7f065c10c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 -- 172.21.15.90:0/1872781951 >> 172.21.15.90:0/1872781951 conn(0x7f065c06be00 msgr2=0x7f065c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:21.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.908+0000 7f06619cc700 1 -- 172.21.15.90:0/1872781951 shutdown_connections 2024-04-06T14:09:21.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.910+0000 7f06619cc700 1 -- 172.21.15.90:0/1872781951 wait complete. 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 Processor -- start 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 -- start start 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 0x7f065c11db20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c117720 0x7f065c117bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f065c118380 con 0x7f065c117720 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f065c1184f0 con 0x7f065c1088b0 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f06619cc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f065c1bb190 con 0x7f065c10cf30 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38912/0 (socket says 172.21.15.90:38912) 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f0653fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 0x7f065c11db20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f065affd700 1 -- 172.21.15.90:0/2474584285 learned_addr learned my addr 172.21.15.90:0/2474584285 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.911+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c117720 0x7f065c117bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 -- 172.21.15.90:0/2474584285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 msgr2=0x7f065c11db20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 0x7f065c11db20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 -- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c117720 msgr2=0x7f065c117bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c117720 0x7f065c117bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 -- 172.21.15.90:0/2474584285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f064c00d650 con 0x7f065c1088b0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.912+0000 7f065affd700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f064c00a850 tx=0x7f064c004410 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.913+0000 7f06609ca700 1 -- 172.21.15.90:0/2474584285 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f064c00acf0 con 0x7f065c1088b0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.913+0000 7f06609ca700 1 -- 172.21.15.90:0/2474584285 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f064c00ae50 con 0x7f065c1088b0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.913+0000 7f06609ca700 1 -- 172.21.15.90:0/2474584285 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f064c01e6b0 con 0x7f065c1088b0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.913+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f065c1bb410 con 0x7f065c1088b0 2024-04-06T14:09:21.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.913+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f065c1bb780 con 0x7f065c1088b0 2024-04-06T14:09:21.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.918+0000 7f06609ca700 1 -- 172.21.15.90:0/2474584285 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f064c0154f0 con 0x7f065c1088b0 2024-04-06T14:09:21.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 Processor -- start 2024-04-06T14:09:21.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 -- start start 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f06609ca700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 0x7f063c0a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9b8b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c51d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca910990 con 0x5603ca9c2de0 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca90e350 con 0x5603ca9b8780 2024-04-06T14:09:21.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f066db80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca909c40 con 0x5603ca9b90a0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.919+0000 7f0653fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f06609ca700 1 -- 172.21.15.90:0/2474584285 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f064c0cda50 con 0x7f065c1088b0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9b8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f0653fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c28a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52824/0 (socket says 172.21.15.90:52824) 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f0653fff700 1 -- 172.21.15.90:0/2084980926 learned_addr learned my addr 172.21.15.90:0/2084980926 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c51d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 0x7f063c0a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 -- 172.21.15.90:0/2084980926 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 msgr2=0x5603ca9c28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:21.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c28a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 -- 172.21.15.90:0/2084980926 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 msgr2=0x5603ca9b8b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:21.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9b8b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.920+0000 7f065b7fe700 1 -- 172.21.15.90:0/2084980926 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5603ca78b3b0 con 0x5603ca9c2de0 2024-04-06T14:09:21.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.921+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 0x7f063c0a38e0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f0654009550 tx=0x7f0654009f70 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:21.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.921+0000 7f065b7fe700 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c51d0 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f065401ab40 tx=0x7f065401d000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ae3d360db0720de server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:21.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.929+0000 7f0651ffb700 1 -- 172.21.15.90:0/2084980926 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0654027070 con 0x5603ca9c2de0 2024-04-06T14:09:21.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.929+0000 7f0651ffb700 1 -- 172.21.15.90:0/2084980926 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f065400f040 con 0x5603ca9c2de0 2024-04-06T14:09:21.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.929+0000 7f0651ffb700 1 -- 172.21.15.90:0/2084980926 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f065400e050 con 0x5603ca9c2de0 2024-04-06T14:09:21.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.930+0000 7f066db80b80 1 -- 172.21.15.90:0/2084980926 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 msgr2=0x5603ca9c51d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:21.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.930+0000 7f066db80b80 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c51d0 secure :-1 s=READY pgs=530 cs=0 l=1 rev1=1 crypto rx=0x7f065401ab40 tx=0x7f065401d000 comp rx=0 tx=0).stop 2024-04-06T14:09:21.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.944+0000 7f066db80b80 1 -- 172.21.15.90:0/2084980926 shutdown_connections 2024-04-06T14:09:21.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.944+0000 7f066db80b80 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c51d0 unknown :-1 s=CLOSED pgs=530 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.944+0000 7f066db80b80 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9b8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.944+0000 7f066db80b80 1 --2- 172.21.15.90:0/2084980926 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.944+0000 7f066db80b80 1 -- 172.21.15.90:0/2084980926 >> 172.21.15.90:0/2084980926 conn(0x5603ca9176a0 msgr2=0x5603ca9b3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:21.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.951+0000 7f066db80b80 1 -- 172.21.15.90:0/2084980926 shutdown_connections 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.961+0000 7f066db80b80 1 -- 172.21.15.90:0/2084980926 wait complete. 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.961+0000 7f066db80b80 1 -- start start 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.961+0000 7f066db80b80 1 Processor -- start 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bfc10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9c2de0 0x5603ca9c0350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca8a5d80 con 0x5603ca9b90a0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca8a5a20 con 0x5603ca9b8780 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca78d6a0 con 0x5603ca9c2de0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f066db80b80 10 client.0 _ll_register_callbacks cb 0x5603ca8af470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38940/0 (socket says 172.21.15.90:38940) 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 -- 172.21.15.90:0/1728027498 learned_addr learned my addr 172.21.15.90:0/1728027498 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 -- 172.21.15.90:0/1728027498 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9c2de0 msgr2=0x5603ca9c0350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9c2de0 0x5603ca9c0350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 -- 172.21.15.90:0/1728027498 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 msgr2=0x5603ca9bfc10 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bfc10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.962+0000 7f065affd700 1 -- 172.21.15.90:0/1728027498 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f065401a900 con 0x5603ca9b8780 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f065affd700 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f064c097100 tx=0x7f064c09a560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=684bafaf60c94a75 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f064affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:21.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f064c06bdc0 con 0x5603ca9b8780 2024-04-06T14:09:21.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f064c09ac00 con 0x5603ca9b8780 2024-04-06T14:09:21.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5603ca8ca380 con 0x5603ca9b8780 2024-04-06T14:09:21.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.966+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f064c09ea30 con 0x5603ca9b8780 2024-04-06T14:09:21.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.967+0000 7f066db80b80 10 client.25745 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.967+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5603ca737b60 con 0x5603ca9b8780 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5603ca8ca380 con 0x5603ca9b8780 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f064c0cb430 con 0x5603ca9b8780 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 10 client.25745 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 10 client.25745 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 20 client.25745 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603ca9f4720 0x5603ca9f6b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 4 client.25745 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5603ca902700 con 0x5603ca9f4720 2024-04-06T14:09:21.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.969+0000 7f0653fff700 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603ca9f4720 0x5603ca9f6b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:21.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.970+0000 7f064affd700 10 client.25745 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:21.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.970+0000 7f0653fff700 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603ca9f4720 0x5603ca9f6b40 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5e72493c48848877 server_cookie=76b7e5426b7b0ebe in_seq=0 out_seq=0 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.970+0000 7f064affd700 1 -- 172.21.15.90:0/1728027498 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5603ca902700 con 0x5603ca9f4720 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.970+0000 7f064affd700 10 client.25745 handle_command_reply: tid=0 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 client.25745 shutdown 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 msgr2=0x5603ca9c2880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f064c097100 tx=0x7f064c09a560 comp rx=0 tx=0).stop 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 shutdown_connections 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603ca9f4720 0x5603ca9f6b40 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9c2de0 0x5603ca9c0350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f064affd700 0 client.25745 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c2880 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 --2- 172.21.15.90:0/1728027498 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bfc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:21.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.971+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 >> 172.21.15.90:0/1728027498 conn(0x5603ca9176a0 msgr2=0x5603ca9b2c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:21.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.973+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 shutdown_connections 2024-04-06T14:09:21.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.973+0000 7f066db80b80 1 -- 172.21.15.90:0/1728027498 wait complete. 2024-04-06T14:09:21.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:21.977+0000 7f066db80b80 20 client.25745 trim_cache size 0 max 16384 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.006+0000 7f066db80b80 1 Processor -- start 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.006+0000 7f066db80b80 1 -- start start 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bebf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9c2de0 0x5603ca9b33a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca9be1a0 con 0x5603ca9b90a0 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca9b9fe0 con 0x5603ca9c2de0 2024-04-06T14:09:22.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f066db80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603ca9ba9c0 con 0x5603ca9b8780 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52852/0 (socket says 172.21.15.90:52852) 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 -- 172.21.15.90:0/1883249739 learned_addr learned my addr 172.21.15.90:0/1883249739 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065b7fe700 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9c2de0 0x5603ca9b33a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 -- 172.21.15.90:0/1883249739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9c2de0 msgr2=0x5603ca9b33a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9c2de0 0x5603ca9b33a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 -- 172.21.15.90:0/1883249739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 msgr2=0x5603ca9bebf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bebf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.007+0000 7f065affd700 1 -- 172.21.15.90:0/1883249739 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5603ca8cfea0 con 0x5603ca9b8780 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f065affd700 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f064c005910 tx=0x7f064c015c60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dbc0fb4846622e5c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f06527fc700 1 -- 172.21.15.90:0/1883249739 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f064c06b240 con 0x5603ca9b8780 2024-04-06T14:09:22.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f06527fc700 1 -- 172.21.15.90:0/1883249739 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f064c09ac00 con 0x5603ca9b8780 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f06527fc700 1 -- 172.21.15.90:0/1883249739 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f064c09a0f0 con 0x5603ca9b8780 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 -- 172.21.15.90:0/1883249739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 msgr2=0x5603ca9c9a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f064c005910 tx=0x7f064c015c60 comp rx=0 tx=0).stop 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 -- 172.21.15.90:0/1883249739 shutdown_connections 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9c2de0 0x5603ca9b33a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b8780 0x5603ca9c9a00 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 --2- 172.21.15.90:0/1883249739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9b90a0 0x5603ca9bebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.008+0000 7f066db80b80 1 -- 172.21.15.90:0/1883249739 >> 172.21.15.90:0/1883249739 conn(0x5603ca9176a0 msgr2=0x5603ca9be4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- 172.21.15.90:0/1883249739 shutdown_connections 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- 172.21.15.90:0/1883249739 wait complete. 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- start start 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 Processor -- start 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c80a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c8b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603caa48540 con 0x5603ca9c2de0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603caa48680 con 0x5603ca9b8780 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.009+0000 7f066db80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5603caa487f0 con 0x5603ca9b90a0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f066db80b80 10 client.0 _ll_register_callbacks cb 0x5603ca8af470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f065affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c80a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52856/0 (socket says 172.21.15.90:52856) 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 -- 172.21.15.90:0/3171247951 learned_addr learned my addr 172.21.15.90:0/3171247951 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f065b7fe700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c8b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 -- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 msgr2=0x5603ca9c80a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c80a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 -- 172.21.15.90:0/3171247951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 msgr2=0x5603ca9c8b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c8b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 -- 172.21.15.90:0/3171247951 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5603ca737b60 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f065affd700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c80a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f0653fff700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x5603ca9f7430 tx=0x7f0644010710 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f4027bf7ce2b8a1e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f064affd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0644017040 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0644010dc0 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.010+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f065400f040 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f064401c5b0 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 10 client.25657 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5603ca78fcd0 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f065400f040 con 0x5603ca9b90a0 2024-04-06T14:09:22.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f064400fb20 con 0x5603ca9b90a0 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 10 client.25657 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 10 client.25657 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 20 client.25657 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603caa4a040 0x5603caa4c460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 4 client.25657 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.011+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5603ca758cf0 con 0x5603caa4a040 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f065affd700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603caa4a040 0x5603caa4c460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f065affd700 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603caa4a040 0x5603caa4c460 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d2b34f26a958fd3e server_cookie=33bb8e6eee1ab173 in_seq=0 out_seq=0 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f064affd700 10 client.25657 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f064affd700 1 -- 172.21.15.90:0/3171247951 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x5603ca758cf0 con 0x5603caa4a040 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f064affd700 10 client.25657 handle_command_reply: tid=0 2024-04-06T14:09:22.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.012+0000 7f066db80b80 1 client.25657 shutdown 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 msgr2=0x5603ca9c85e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x5603ca9f7430 tx=0x7f0644010710 comp rx=0 tx=0).stop 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 shutdown_connections 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5603ca9c2de0 0x5603ca9c8b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5603ca9b8780 0x5603ca9c80a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5603caa4a040 0x5603caa4c460 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 --2- 172.21.15.90:0/3171247951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5603ca9b90a0 0x5603ca9c85e0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.013+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 >> 172.21.15.90:0/3171247951 conn(0x5603ca9176a0 msgr2=0x5603ca9b7770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.014+0000 7f064affd700 0 client.25657 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.014+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 shutdown_connections 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.014+0000 7f066db80b80 1 -- 172.21.15.90:0/3171247951 wait complete. 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.015+0000 7f066db80b80 20 client.25657 trim_cache size 0 max 16384 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:09:22.018 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:22.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 msgr2=0x7f063c0a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 0x7f063c0a38e0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f0654009550 tx=0x7f0654009f70 comp rx=0 tx=0).stop 2024-04-06T14:09:22.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 msgr2=0x7f065c11d5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f064c00a850 tx=0x7f064c004410 comp rx=0 tx=0).stop 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 shutdown_connections 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f063c0a1420 0x7f063c0a38e0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f065c117720 0x7f065c117bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f065c1088b0 0x7f065c11d5e0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 --2- 172.21.15.90:0/2474584285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f065c10cf30 0x7f065c11db20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 >> 172.21.15.90:0/2474584285 conn(0x7f065c06be00 msgr2=0x7f065c070d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 shutdown_connections 2024-04-06T14:09:22.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.018+0000 7f06619cc700 1 -- 172.21.15.90:0/2474584285 wait complete. 2024-04-06T14:09:22.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:21 smithi090 ceph-mon[43420]: pgmap v122: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 26 MiB/s wr, 1.20k op/s 2024-04-06T14:09:22.033 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:09:22.033 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:09:22.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:21 smithi134 ceph-mon[47549]: pgmap v122: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 26 MiB/s wr, 1.20k op/s 2024-04-06T14:09:22.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.218+0000 7f0d449b1700 1 -- 172.21.15.90:0/26947591 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d401074d0 msgr2=0x7f0d40107950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.218+0000 7f0d449b1700 1 --2- 172.21.15.90:0/26947591 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d401074d0 0x7f0d40107950 secure :-1 s=READY pgs=531 cs=0 l=1 rev1=1 crypto rx=0x7f0d34009990 tx=0x7f0d34009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.219+0000 7f0d449b1700 1 -- 172.21.15.90:0/26947591 shutdown_connections 2024-04-06T14:09:22.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.219+0000 7f0d449b1700 1 --2- 172.21.15.90:0/26947591 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d40107e90 0x7f0d401140b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.219+0000 7f0d449b1700 1 --2- 172.21.15.90:0/26947591 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d4010c240 0x7f0d4010c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.219+0000 7f0d449b1700 1 --2- 172.21.15.90:0/26947591 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d401074d0 0x7f0d40107950 unknown :-1 s=CLOSED pgs=531 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.219+0000 7f0d449b1700 1 -- 172.21.15.90:0/26947591 >> 172.21.15.90:0/26947591 conn(0x7f0d40076c20 msgr2=0x7f0d40105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.222+0000 7f0d449b1700 1 -- 172.21.15.90:0/26947591 shutdown_connections 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.222+0000 7f0d449b1700 1 -- 172.21.15.90:0/26947591 wait complete. 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.223+0000 7f0d449b1700 1 Processor -- start 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.223+0000 7f0d449b1700 1 -- start start 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.223+0000 7f0d449b1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.223+0000 7f0d449b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 0x7f0d4019dc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.223+0000 7f0d449b1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d4010c240 0x7f0d401a1fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d449b1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d401a25f0 con 0x7f0d40107e90 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d449b1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d401a2760 con 0x7f0d4010c240 2024-04-06T14:09:22.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d449b1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d401a2920 con 0x7f0d401074d0 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52866/0 (socket says 172.21.15.90:52866) 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 -- 172.21.15.90:0/2962425947 learned_addr learned my addr 172.21.15.90:0/2962425947 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3ed9d700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d4010c240 0x7f0d401a1fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3dd9b700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 0x7f0d4019dc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 -- 172.21.15.90:0/2962425947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d4010c240 msgr2=0x7f0d401a1fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d4010c240 0x7f0d401a1fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 -- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 msgr2=0x7f0d4019dc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 0x7f0d4019dc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3e59c700 1 -- 172.21.15.90:0/2962425947 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d340095f0 con 0x7f0d401074d0 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.224+0000 7f0d3dd9b700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 0x7f0d4019dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:09:22.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d3e59c700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0d3800ef10 tx=0x7f0d3800c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d3800cad0 con 0x7f0d401074d0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d3800f040 con 0x7f0d401074d0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d38007ea0 con 0x7f0d401074d0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d401a2ba0 con 0x7f0d401074d0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.225+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d401aa440 con 0x7f0d401074d0 2024-04-06T14:09:22.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.226+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d4010f750 con 0x7f0d401074d0 2024-04-06T14:09:22.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.227+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0d380146e0 con 0x7f0d401074d0 2024-04-06T14:09:22.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.228+0000 7f0d2f7fe700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 0x7f0d280a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.228+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0d380cdbe0 con 0x7f0d401074d0 2024-04-06T14:09:22.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.230+0000 7f0d3dd9b700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 0x7f0d280a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.231+0000 7f0d3dd9b700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 0x7f0d280a37d0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f0d3400b2e0 tx=0x7f0d34005e80 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.232+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0d38097000 con 0x7f0d401074d0 2024-04-06T14:09:22.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:21 smithi105 ceph-mon[46969]: pgmap v122: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 26 MiB/s wr, 1.20k op/s 2024-04-06T14:09:22.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.620+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0d40108990 con 0x7f0d401074d0 2024-04-06T14:09:22.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.621+0000 7f0d2f7fe700 1 -- 172.21.15.90:0/2962425947 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f0d3809a620 con 0x7f0d401074d0 2024-04-06T14:09:22.621 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:09:22.621 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:09:22.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.624+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 msgr2=0x7f0d280a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.624+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 0x7f0d280a37d0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f0d3400b2e0 tx=0x7f0d34005e80 comp rx=0 tx=0).stop 2024-04-06T14:09:22.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.624+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 msgr2=0x7f0d4019d6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.624+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0d3800ef10 tx=0x7f0d3800c3a0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 shutdown_connections 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0d280a1310 0x7f0d280a37d0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0d4010c240 0x7f0d401a1fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0d401074d0 0x7f0d4019d6d0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 --2- 172.21.15.90:0/2962425947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0d40107e90 0x7f0d4019dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.629+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 >> 172.21.15.90:0/2962425947 conn(0x7f0d40076c20 msgr2=0x7f0d40077d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.631+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 shutdown_connections 2024-04-06T14:09:22.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.632+0000 7f0d449b1700 1 -- 172.21.15.90:0/2962425947 wait complete. 2024-04-06T14:09:22.633 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:09:22.650 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:22.650 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:09:22.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 -- 172.21.15.90:0/3211808121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218109750 msgr2=0x7f4218109b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 --2- 172.21.15.90:0/3211808121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218109750 0x7f4218109b30 secure :-1 s=READY pgs=532 cs=0 l=1 rev1=1 crypto rx=0x7f4214009990 tx=0x7f4214009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 -- 172.21.15.90:0/3211808121 shutdown_connections 2024-04-06T14:09:22.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 --2- 172.21.15.90:0/3211808121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4218108640 0x7f4218108aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 --2- 172.21.15.90:0/3211808121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218109750 0x7f4218109b30 unknown :-1 s=CLOSED pgs=532 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 --2- 172.21.15.90:0/3211808121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f421810a070 0x7f42181127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.855+0000 7f421f372700 1 -- 172.21.15.90:0/3211808121 >> 172.21.15.90:0/3211808121 conn(0x7f421806be00 msgr2=0x7f421806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.857+0000 7f421f372700 1 -- 172.21.15.90:0/3211808121 shutdown_connections 2024-04-06T14:09:22.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 -- 172.21.15.90:0/3211808121 wait complete. 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 Processor -- start 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 -- start start 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 0x7f4218123560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f421810a070 0x7f421811ded0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4218118b40 con 0x7f4218108640 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4218118cb0 con 0x7f4218109750 2024-04-06T14:09:22.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.858+0000 7f421f372700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4218118e70 con 0x7f421810a070 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421d10e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421d10e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:32936/0 (socket says 172.21.15.90:32936) 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421d10e700 1 -- 172.21.15.90:0/268088455 learned_addr learned my addr 172.21.15.90:0/268088455 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421d90f700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f421810a070 0x7f421811ded0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 0x7f4218123560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 -- 172.21.15.90:0/268088455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f421810a070 msgr2=0x7f421811ded0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f421810a070 0x7f421811ded0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 -- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 msgr2=0x7f421811d7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.859+0000 7f421c90d700 1 -- 172.21.15.90:0/268088455 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42140095f0 con 0x7f4218109750 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f421d10e700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f421c90d700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 0x7f4218123560 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f420800d950 tx=0x7f420800dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f420e7fc700 1 -- 172.21.15.90:0/268088455 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f420800f890 con 0x7f4218109750 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f420e7fc700 1 -- 172.21.15.90:0/268088455 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4208010460 con 0x7f4218109750 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f420e7fc700 1 -- 172.21.15.90:0/268088455 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f420800e610 con 0x7f4218109750 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4218119150 con 0x7f4218109750 2024-04-06T14:09:22.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.860+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4218119610 con 0x7f4218109750 2024-04-06T14:09:22.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 Processor -- start 2024-04-06T14:09:22.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 -- start start 2024-04-06T14:09:22.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a370b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a37d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a2c8990 con 0x56172a37ade0 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a2c6350 con 0x56172a370780 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.865+0000 7f422b526b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a2c1c40 con 0x56172a3710a0 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52902/0 (socket says 172.21.15.90:52902) 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 -- 172.21.15.90:0/1131773885 learned_addr learned my addr 172.21.15.90:0/1131773885 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d90f700 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a370b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 -- 172.21.15.90:0/1131773885 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 msgr2=0x56172a370b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a370b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 -- 172.21.15.90:0/1131773885 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 msgr2=0x56172a37d1d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:22.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a37d1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 -- 172.21.15.90:0/1131773885 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56172a1433b0 con 0x56172a3710a0 2024-04-06T14:09:22.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.866+0000 7f421d10e700 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4214005ef0 tx=0x7f4214005040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f0498ed6995ed61d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.867+0000 7f420e7fc700 1 -- 172.21.15.90:0/268088455 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4208010890 con 0x7f4218109750 2024-04-06T14:09:22.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.867+0000 7f42037fe700 1 -- 172.21.15.90:0/1131773885 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f421401d070 con 0x56172a3710a0 2024-04-06T14:09:22.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.867+0000 7f42037fe700 1 -- 172.21.15.90:0/1131773885 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f421401e040 con 0x56172a3710a0 2024-04-06T14:09:22.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.867+0000 7f42037fe700 1 -- 172.21.15.90:0/1131773885 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f421400bcc0 con 0x56172a3710a0 2024-04-06T14:09:22.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f420e7fc700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 0x7f42040a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f420e7fc700 1 -- 172.21.15.90:0/268088455 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f42080cc720 con 0x7f4218109750 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 -- 172.21.15.90:0/1131773885 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 msgr2=0x56172a37a8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4214005ef0 tx=0x7f4214005040 comp rx=0 tx=0).stop 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 -- 172.21.15.90:0/1131773885 shutdown_connections 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a37d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a370b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 --2- 172.21.15.90:0/1131773885 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37a8a0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 -- 172.21.15.90:0/1131773885 >> 172.21.15.90:0/1131773885 conn(0x56172a2cf6a0 msgr2=0x56172a36ba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 -- 172.21.15.90:0/1131773885 shutdown_connections 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.868+0000 7f422b526b80 1 -- 172.21.15.90:0/1131773885 wait complete. 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.869+0000 7f422b526b80 1 -- start start 2024-04-06T14:09:22.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.869+0000 7f422b526b80 1 Processor -- start 2024-04-06T14:09:22.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a37a830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a377b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a25dd80 con 0x56172a37ade0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a25da20 con 0x56172a370780 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a1456a0 con 0x56172a3710a0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f422b526b80 10 client.0 _ll_register_callbacks cb 0x56172a267470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:32952/0 (socket says 172.21.15.90:32952) 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 -- 172.21.15.90:0/2634751102 learned_addr learned my addr 172.21.15.90:0/2634751102 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d10e700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a37a830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 0x7f42040a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421c90d700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a377b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 -- 172.21.15.90:0/2634751102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 msgr2=0x56172a377b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a377b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 -- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 msgr2=0x56172a37a830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a37a830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.870+0000 7f421d90f700 1 -- 172.21.15.90:0/2634751102 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42181094d0 con 0x56172a37ade0 2024-04-06T14:09:22.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.871+0000 7f421d90f700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 0x7f42040a3850 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f42100196b0 tx=0x7f4210021000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.871+0000 7f421d90f700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f421001bb30 tx=0x7f421001be40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8dd9c3eaf0363ffa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.871+0000 7f4200ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.872+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4210020070 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.872+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4210016650 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.872+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56172a282380 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.873+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f421000e040 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.873+0000 7f422b526b80 10 client.16071 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.873+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56172a0efb60 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.873+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56172a282380 con 0x56172a37ade0 2024-04-06T14:09:22.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f421002d5a0 con 0x56172a37ade0 2024-04-06T14:09:22.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 10 client.16071 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:22.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 10 client.16071 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:09:22.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 20 client.16071 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:22.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a38aa80 0x56172a38cea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 4 client.16071 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56172a2ba700 con 0x56172a38aa80 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f421d10e700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a38aa80 0x56172a38cea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f4200ff9700 10 client.16071 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.874+0000 7f421d10e700 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a38aa80 0x56172a38cea0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=bae3e9422641419e server_cookie=e2c50f98e3f54653 in_seq=0 out_seq=0 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.875+0000 7f4200ff9700 1 -- 172.21.15.90:0/2634751102 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56172a2ba700 con 0x56172a38aa80 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.875+0000 7f4200ff9700 10 client.16071 handle_command_reply: tid=0 2024-04-06T14:09:22.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.875+0000 7f422b526b80 1 client.16071 shutdown 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.875+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 msgr2=0x56172a3782b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.875+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 secure :-1 s=READY pgs=533 cs=0 l=1 rev1=1 crypto rx=0x7f421001bb30 tx=0x7f421001be40 comp rx=0 tx=0).stop 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 shutdown_connections 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a38aa80 0x56172a38cea0 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a37ade0 0x56172a3782b0 unknown :-1 s=CLOSED pgs=533 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a370780 0x56172a37a830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 --2- 172.21.15.90:0/2634751102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a377b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 >> 172.21.15.90:0/2634751102 conn(0x56172a2cf6a0 msgr2=0x56172a36ac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.876+0000 7f4200ff9700 0 client.16071 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:22.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.877+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 shutdown_connections 2024-04-06T14:09:22.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.877+0000 7f422b526b80 1 -- 172.21.15.90:0/2634751102 wait complete. 2024-04-06T14:09:22.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.877+0000 7f422b526b80 20 client.16071 trim_cache size 0 max 16384 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.899+0000 7f422b526b80 1 Processor -- start 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.899+0000 7f422b526b80 1 -- start start 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a3710a0 0x56172a381a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a372e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a372690 con 0x56172a3710a0 2024-04-06T14:09:22.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a371a10 con 0x56172a37ade0 2024-04-06T14:09:22.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f422b526b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a38d230 con 0x56172a370780 2024-04-06T14:09:22.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52924/0 (socket says 172.21.15.90:52924) 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 -- 172.21.15.90:0/2139643240 learned_addr learned my addr 172.21.15.90:0/2139643240 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d90f700 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a372e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 -- 172.21.15.90:0/2139643240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 msgr2=0x56172a372e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a372e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 -- 172.21.15.90:0/2139643240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a3710a0 msgr2=0x56172a381a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a3710a0 0x56172a381a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.900+0000 7f421d10e700 1 -- 172.21.15.90:0/2139643240 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56172a287ea0 con 0x56172a370780 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.901+0000 7f421d10e700 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f421400b2e0 tx=0x7f421401a040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6d4725377e9de0e1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.901+0000 7f4203fff700 1 -- 172.21.15.90:0/2139643240 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f421401d040 con 0x56172a370780 2024-04-06T14:09:22.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.901+0000 7f4203fff700 1 -- 172.21.15.90:0/2139643240 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4214003ef0 con 0x56172a370780 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 -- 172.21.15.90:0/2139643240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 msgr2=0x56172a378230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f421400b2e0 tx=0x7f421401a040 comp rx=0 tx=0).stop 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 -- 172.21.15.90:0/2139643240 shutdown_connections 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a372e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a370780 0x56172a378230 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 --2- 172.21.15.90:0/2139643240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a3710a0 0x56172a381a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 -- 172.21.15.90:0/2139643240 >> 172.21.15.90:0/2139643240 conn(0x56172a2cf6a0 msgr2=0x56172a367890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 -- 172.21.15.90:0/2139643240 shutdown_connections 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.902+0000 7f422b526b80 1 -- 172.21.15.90:0/2139643240 wait complete. 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 -- start start 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 Processor -- start 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a370780 0x56172a37f590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a3dd8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a380270 con 0x56172a370780 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a3dde20 con 0x56172a37ade0 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56172a3ddfe0 con 0x56172a3710a0 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.903+0000 7f422b526b80 10 client.0 _ll_register_callbacks cb 0x56172a267470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52938/0 (socket says 172.21.15.90:52938) 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 -- 172.21.15.90:0/3639449198 learned_addr learned my addr 172.21.15.90:0/3639449198 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421d90f700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a3dd8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 -- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 msgr2=0x56172a3dd8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a3dd8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 -- 172.21.15.90:0/3639449198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a370780 msgr2=0x56172a37f590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a370780 0x56172a37f590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.904+0000 7f421c90d700 1 -- 172.21.15.90:0/3639449198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56172a1433b0 con 0x56172a3710a0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f421c90d700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f420800dfb0 tx=0x7f4208099160 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5b2a3913f56b1fe4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f4200ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f420809d630 con 0x56172a3710a0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f42080a1c80 con 0x56172a3710a0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f421401e040 con 0x56172a3710a0 2024-04-06T14:09:22.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.905+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f42080ca0c0 con 0x56172a3710a0 2024-04-06T14:09:22.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 10 client.25681 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:22.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56172a147cd0 con 0x56172a3710a0 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f421401e040 con 0x56172a3710a0 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4208098cf0 con 0x56172a3710a0 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 10 client.25681 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 10 client.25681 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 20 client.25681 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a3df730 0x56172a3e1b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 4 client.25681 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.908+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x56172a110cf0 con 0x56172a3df730 2024-04-06T14:09:22.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.909+0000 7f421d10e700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a3df730 0x56172a3e1b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.909+0000 7f4200ff9700 10 client.25681 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.909+0000 7f421d10e700 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a3df730 0x56172a3e1b50 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=1d59b15d61b07d49 server_cookie=f4d398940c41a40c in_seq=0 out_seq=0 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f4200ff9700 1 -- 172.21.15.90:0/3639449198 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x56172a110cf0 con 0x56172a3df730 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f4200ff9700 10 client.25681 handle_command_reply: tid=0 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 client.25681 shutdown 2024-04-06T14:09:22.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 msgr2=0x56172a37fad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f420800dfb0 tx=0x7f4208099160 comp rx=0 tx=0).stop 2024-04-06T14:09:22.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 shutdown_connections 2024-04-06T14:09:22.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x56172a3df730 0x56172a3e1b50 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56172a37ade0 0x56172a3dd8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56172a370780 0x56172a37f590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 --2- 172.21.15.90:0/3639449198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56172a3710a0 0x56172a37fad0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 >> 172.21.15.90:0/3639449198 conn(0x56172a2cf6a0 msgr2=0x56172a372e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.910+0000 7f4200ff9700 0 client.25681 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.911+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 shutdown_connections 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.911+0000 7f422b526b80 1 -- 172.21.15.90:0/3639449198 wait complete. 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.911+0000 7f422b526b80 20 client.25681 trim_cache size 0 max 16384 2024-04-06T14:09:22.911 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:22.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 msgr2=0x7f42040a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 0x7f42040a3850 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f42100196b0 tx=0x7f4210021000 comp rx=0 tx=0).stop 2024-04-06T14:09:22.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 msgr2=0x7f4218123560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:22.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 0x7f4218123560 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f420800d950 tx=0x7f420800dc60 comp rx=0 tx=0).stop 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 shutdown_connections 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f42040a1390 0x7f42040a3850 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f421810a070 0x7f421811ded0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.912+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4218108640 0x7f421811d7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.913+0000 7f421f372700 1 --2- 172.21.15.90:0/268088455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4218109750 0x7f4218123560 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.913+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 >> 172.21.15.90:0/268088455 conn(0x7f421806be00 msgr2=0x7f421806fa10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.913+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 shutdown_connections 2024-04-06T14:09:22.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:22.913+0000 7f421f372700 1 -- 172.21.15.90:0/268088455 wait complete. 2024-04-06T14:09:22.943 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:22.943 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:09:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 215.0 s, 2513.8 tps, lat 39.744 ms stddev 45.154 2024-04-06T14:09:23.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.175+0000 7f15d45c4700 1 -- 172.21.15.90:0/3682740647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 msgr2=0x7f15cc1082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.175+0000 7f15d45c4700 1 --2- 172.21.15.90:0/3682740647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 0x7f15cc1082f0 secure :-1 s=READY pgs=534 cs=0 l=1 rev1=1 crypto rx=0x7f15bc009990 tx=0x7f15bc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.179+0000 7f15d45c4700 1 -- 172.21.15.90:0/3682740647 shutdown_connections 2024-04-06T14:09:23.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.179+0000 7f15d45c4700 1 --2- 172.21.15.90:0/3682740647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc11a590 0x7f15cc11c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.179+0000 7f15d45c4700 1 --2- 172.21.15.90:0/3682740647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc10f4b0 0x7f15cc107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.179+0000 7f15d45c4700 1 --2- 172.21.15.90:0/3682740647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 0x7f15cc1082f0 unknown :-1 s=CLOSED pgs=534 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.179+0000 7f15d45c4700 1 -- 172.21.15.90:0/3682740647 >> 172.21.15.90:0/3682740647 conn(0x7f15cc06be00 msgr2=0x7f15cc06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.185+0000 7f15d45c4700 1 -- 172.21.15.90:0/3682740647 shutdown_connections 2024-04-06T14:09:23.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.185+0000 7f15d45c4700 1 -- 172.21.15.90:0/3682740647 wait complete. 2024-04-06T14:09:23.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.186+0000 7f15d45c4700 1 Processor -- start 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.186+0000 7f15d45c4700 1 -- start start 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.186+0000 7f15d45c4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc10f4b0 0x7f15cc118a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d45c4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 0x7f15cc118f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d45c4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d45c4700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15cc112390 con 0x7f15cc10f920 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d45c4700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15cc112500 con 0x7f15cc10f4b0 2024-04-06T14:09:23.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d45c4700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15cc1126c0 con 0x7f15cc11a590 2024-04-06T14:09:23.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52962/0 (socket says 172.21.15.90:52962) 2024-04-06T14:09:23.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 -- 172.21.15.90:0/1199811177 learned_addr learned my addr 172.21.15.90:0/1199811177 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2360700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc10f4b0 0x7f15cc118a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 -- 172.21.15.90:0/1199811177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc10f4b0 msgr2=0x7f15cc118a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc10f4b0 0x7f15cc118a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 -- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 msgr2=0x7f15cc118f80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 0x7f15cc118f80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.187+0000 7f15d2b61700 1 -- 172.21.15.90:0/1199811177 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15bc0095f0 con 0x7f15cc11a590 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15d2b61700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f15c800b480 tx=0x7f15c800b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15c37fe700 1 -- 172.21.15.90:0/1199811177 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c8011830 con 0x7f15cc11a590 2024-04-06T14:09:23.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15c37fe700 1 -- 172.21.15.90:0/1199811177 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15c8011e50 con 0x7f15cc11a590 2024-04-06T14:09:23.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15cc116590 con 0x7f15cc11a590 2024-04-06T14:09:23.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15c37fe700 1 -- 172.21.15.90:0/1199811177 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c8010e70 con 0x7f15cc11a590 2024-04-06T14:09:23.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.188+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15cc116a50 con 0x7f15cc11a590 2024-04-06T14:09:23.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.190+0000 7f15c37fe700 1 -- 172.21.15.90:0/1199811177 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f15c8010510 con 0x7f15cc11a590 2024-04-06T14:09:23.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.191+0000 7f15c37fe700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 0x7f15b80a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.191+0000 7f15c37fe700 1 -- 172.21.15.90:0/1199811177 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f15c80cc660 con 0x7f15cc11a590 2024-04-06T14:09:23.191 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.192+0000 7f15d2360700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 0x7f15b80a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.192+0000 7f15d2360700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 0x7f15b80a3790 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f15c4009fd0 tx=0x7f15c400d040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.192+0000 7f15e0778b80 1 Processor -- start 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.192+0000 7f15e0778b80 1 -- start start 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786941780 0x556786941b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5567869420a0 0x55678694b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786899990 con 0x5567869420a0 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786897350 con 0x556786941780 2024-04-06T14:09:23.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15e0778b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786892c40 con 0x55678694bde0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52964/0 (socket says 172.21.15.90:52964) 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 -- 172.21.15.90:0/4262422512 learned_addr learned my addr 172.21.15.90:0/4262422512 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2360700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786941780 0x556786941b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 -- 172.21.15.90:0/4262422512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786941780 msgr2=0x556786941b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786941780 0x556786941b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 -- 172.21.15.90:0/4262422512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5567869420a0 msgr2=0x55678694b8a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5567869420a0 0x55678694b8a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 -- 172.21.15.90:0/4262422512 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5567867143b0 con 0x55678694bde0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15d2b61700 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f15c80962c0 tx=0x7f15c8095cc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=632bc3c30fd50d56 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.193+0000 7f15c0ff9700 1 -- 172.21.15.90:0/4262422512 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c8099850 con 0x55678694bde0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15c0ff9700 1 -- 172.21.15.90:0/4262422512 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15c809d680 con 0x55678694bde0 2024-04-06T14:09:23.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15c0ff9700 1 -- 172.21.15.90:0/4262422512 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c8098fa0 con 0x55678694bde0 2024-04-06T14:09:23.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 -- 172.21.15.90:0/4262422512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 msgr2=0x55678694e1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f15c80962c0 tx=0x7f15c8095cc0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 -- 172.21.15.90:0/4262422512 shutdown_connections 2024-04-06T14:09:23.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x55678694e1d0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786941780 0x556786941b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 --2- 172.21.15.90:0/4262422512 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5567869420a0 0x55678694b8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.194+0000 7f15e0778b80 1 -- 172.21.15.90:0/4262422512 >> 172.21.15.90:0/4262422512 conn(0x5567868a06a0 msgr2=0x55678693ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.195+0000 7f15e0778b80 1 -- 172.21.15.90:0/4262422512 shutdown_connections 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.195+0000 7f15e0778b80 1 -- 172.21.15.90:0/4262422512 wait complete. 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.195+0000 7f15e0778b80 1 -- start start 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.195+0000 7f15e0778b80 1 Processor -- start 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x5567869491d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 0x556786957ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55678682ed80 con 0x556786941780 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55678682ea20 con 0x55678694bde0 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567867166a0 con 0x556786949710 2024-04-06T14:09:23.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15e0778b80 10 client.0 _ll_register_callbacks cb 0x556786838470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:32980/0 (socket says 172.21.15.90:32980) 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 -- 172.21.15.90:0/524645112 learned_addr learned my addr 172.21.15.90:0/524645112 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2b61700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 0x556786957ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 -- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 msgr2=0x556786957ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 0x556786957ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 -- 172.21.15.90:0/524645112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 msgr2=0x5567869491d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x5567869491d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 -- 172.21.15.90:0/524645112 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15cc116a50 con 0x556786941780 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2b61700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 0x556786957ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15d2360700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f15c4016cb0 tx=0x7f15c4004b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=96fdc7ef82b050b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.196+0000 7f15b5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:23.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c4010040 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f15c4022440 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x556786853380 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f15c4016e50 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 10 client.16077 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5567866c0b60 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x556786853380 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f15c402c770 con 0x556786941780 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 10 client.16077 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 10 client.16077 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 20 client.16077 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55678695bfd0 0x55678695e3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 4 client.16077 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:23.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.197+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55678688b700 con 0x55678695bfd0 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.198+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55678695bfd0 0x55678695e3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.199+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55678695bfd0 0x55678695e3f0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=50540dea3b6c1ca2 server_cookie=8d2db345b1f4bef2 in_seq=0 out_seq=0 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.199+0000 7f15b5ffb700 10 client.16077 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.199+0000 7f15b5ffb700 1 -- 172.21.15.90:0/524645112 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55678688b700 con 0x55678695bfd0 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.199+0000 7f15b5ffb700 10 client.16077 handle_command_reply: tid=0 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.199+0000 7f15e0778b80 1 client.16077 shutdown 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 msgr2=0x556786948c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 secure :-1 s=READY pgs=535 cs=0 l=1 rev1=1 crypto rx=0x7f15c4016cb0 tx=0x7f15c4004b60 comp rx=0 tx=0).stop 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 shutdown_connections 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55678695bfd0 0x55678695e3f0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786949710 0x556786957ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x556786948c90 unknown :-1 s=CLOSED pgs=535 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 --2- 172.21.15.90:0/524645112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x5567869491d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 >> 172.21.15.90:0/524645112 conn(0x5567868a06a0 msgr2=0x55678693bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15b5ffb700 0 client.16077 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 shutdown_connections 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 1 -- 172.21.15.90:0/524645112 wait complete. 2024-04-06T14:09:23.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.200+0000 7f15e0778b80 20 client.16077 trim_cache size 0 max 16384 2024-04-06T14:09:23.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 Processor -- start 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 -- start start 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786941780 0x5567869529f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x556786946470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55678694e7a0 con 0x556786949710 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567869342f0 con 0x55678694bde0 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15e0778b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786949530 con 0x556786941780 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15d1b5f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15d1b5f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:32996/0 (socket says 172.21.15.90:32996) 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15d1b5f700 1 -- 172.21.15.90:0/2914516191 learned_addr learned my addr 172.21.15.90:0/2914516191 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15d2b61700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x556786946470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.213+0000 7f15d2360700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786941780 0x5567869529f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 -- 172.21.15.90:0/2914516191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786941780 msgr2=0x5567869529f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786941780 0x5567869529f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 -- 172.21.15.90:0/2914516191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 msgr2=0x556786946470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x556786946470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 -- 172.21.15.90:0/2914516191 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556786858ea0 con 0x556786949710 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d2b61700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x556786946470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f15cc10fd40 tx=0x7f15bc001e70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8aa488a4c57e06ce server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15c17fa700 1 -- 172.21.15.90:0/2914516191 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15bc01d070 con 0x556786949710 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15c17fa700 1 -- 172.21.15.90:0/2914516191 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15bc01e040 con 0x556786949710 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.214+0000 7f15c17fa700 1 -- 172.21.15.90:0/2914516191 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15bc021730 con 0x556786949710 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- 172.21.15.90:0/2914516191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 msgr2=0x556786945f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 secure :-1 s=READY pgs=536 cs=0 l=1 rev1=1 crypto rx=0x7f15cc10fd40 tx=0x7f15bc001e70 comp rx=0 tx=0).stop 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- 172.21.15.90:0/2914516191 shutdown_connections 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55678694bde0 0x556786946470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556786941780 0x5567869529f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- 172.21.15.90:0/2914516191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786949710 0x556786945f30 unknown :-1 s=CLOSED pgs=536 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- 172.21.15.90:0/2914516191 >> 172.21.15.90:0/2914516191 conn(0x5567868a06a0 msgr2=0x556786947580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- 172.21.15.90:0/2914516191 shutdown_connections 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- 172.21.15.90:0/2914516191 wait complete. 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- start start 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 Processor -- start 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 0x5567869a9740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x5567869509e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786950fb0 con 0x556786941780 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556786951120 con 0x556786949710 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567869512e0 con 0x55678694bde0 2024-04-06T14:09:23.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15e0778b80 10 client.0 _ll_register_callbacks cb 0x7f15c4016860 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15d2360700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15d2360700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:33002/0 (socket says 172.21.15.90:33002) 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15d2360700 1 -- 172.21.15.90:0/3265838830 learned_addr learned my addr 172.21.15.90:0/3265838830 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 0x5567869a9740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.215+0000 7f15d2b61700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x5567869509e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 -- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 msgr2=0x5567869509e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x5567869509e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 -- 172.21.15.90:0/3265838830 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 msgr2=0x5567869a9740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 0x5567869a9740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 -- 172.21.15.90:0/3265838830 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5567867143b0 con 0x556786941780 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 0x5567869a9740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15d2360700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f15c4009e00 tx=0x7f15c4007190 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa31b3d7a167fe8f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15b5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15c4010030 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f15c4003ca0 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f15c809d680 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f15c4003e60 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15e0778b80 10 client.16083 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f15cc11d270 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.216+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15c809d680 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f15c4044070 con 0x556786941780 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 10 client.16083 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 10 client.16083 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 20 client.16083 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5567869b1010 0x5567869b3430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 4 client.16083 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5567866e1cf0 con 0x5567869b1010 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5567869b1010 0x5567869b3430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15b5ffb700 10 client.16083 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15d1b5f700 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5567869b1010 0x5567869b3430 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=5f353dd144077b51 server_cookie=140de152e650f137 in_seq=0 out_seq=0 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15b5ffb700 1 -- 172.21.15.90:0/3265838830 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5567866e1cf0 con 0x5567869b1010 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15b5ffb700 10 client.16083 handle_command_reply: tid=0 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 client.16083 shutdown 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 msgr2=0x5567869a9200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 secure :-1 s=READY pgs=537 cs=0 l=1 rev1=1 crypto rx=0x7f15c4009e00 tx=0x7f15c4007190 comp rx=0 tx=0).stop 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.217+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 shutdown_connections 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5567869b1010 0x5567869b3430 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55678694bde0 0x5567869509e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556786941780 0x5567869a9200 unknown :-1 s=CLOSED pgs=537 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 --2- 172.21.15.90:0/3265838830 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556786949710 0x5567869a9740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15b5ffb700 0 client.16083 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 >> 172.21.15.90:0/3265838830 conn(0x5567868a06a0 msgr2=0x556786941000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 shutdown_connections 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 1 -- 172.21.15.90:0/3265838830 wait complete. 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15e0778b80 20 client.16083 trim_cache size 0 max 16384 2024-04-06T14:09:23.224 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 msgr2=0x7f15b80a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 0x7f15b80a3790 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f15c4009fd0 tx=0x7f15c400d040 comp rx=0 tx=0).stop 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 msgr2=0x7f15cc111bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f15c800b480 tx=0x7f15c800b790 comp rx=0 tx=0).stop 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 shutdown_connections 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15b80a12d0 0x7f15b80a3790 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15cc11a590 0x7f15cc111bc0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15cc10f4b0 0x7f15cc118a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 --2- 172.21.15.90:0/1199811177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15cc10f920 0x7f15cc118f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 >> 172.21.15.90:0/1199811177 conn(0x7f15cc06be00 msgr2=0x7f15cc06f7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 shutdown_connections 2024-04-06T14:09:23.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.218+0000 7f15d45c4700 1 -- 172.21.15.90:0/1199811177 wait complete. 2024-04-06T14:09:23.229 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:23.230 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:09:23.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 -- 172.21.15.90:0/1137932386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 msgr2=0x7f8040068fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 --2- 172.21.15.90:0/1137932386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f8040068fa0 secure :-1 s=READY pgs=538 cs=0 l=1 rev1=1 crypto rx=0x7f8034009990 tx=0x7f8034009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 -- 172.21.15.90:0/1137932386 shutdown_connections 2024-04-06T14:09:23.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 --2- 172.21.15.90:0/1137932386 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8040103770 0x7f8040103bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 --2- 172.21.15.90:0/1137932386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f8040068fa0 unknown :-1 s=CLOSED pgs=538 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 --2- 172.21.15.90:0/1137932386 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f80401060e0 0x7f80400694e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 -- 172.21.15.90:0/1137932386 >> 172.21.15.90:0/1137932386 conn(0x7f80400769b0 msgr2=0x7f8040076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.425+0000 7f8046d7d700 1 -- 172.21.15.90:0/1137932386 shutdown_connections 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.426+0000 7f8046d7d700 1 -- 172.21.15.90:0/1137932386 wait complete. 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.426+0000 7f8046d7d700 1 Processor -- start 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 -- start start 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f80401146f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80401060e0 0x7f804010d340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f804010db10 con 0x7f8040105c70 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f804010dc80 con 0x7f8040103770 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8046d7d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f804010de40 con 0x7f80401060e0 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8044b19700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8044b19700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:56232/0 (socket says 172.21.15.90:56232) 2024-04-06T14:09:23.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f8044b19700 1 -- 172.21.15.90:0/867771505 learned_addr learned my addr 172.21.15.90:0/867771505 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f803ffff700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f80401146f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.427+0000 7f804531a700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80401060e0 0x7f804010d340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 -- 172.21.15.90:0/867771505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80401060e0 msgr2=0x7f804010d340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80401060e0 0x7f804010d340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 -- 172.21.15.90:0/867771505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 msgr2=0x7f8040114170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 -- 172.21.15.90:0/867771505 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80340095f0 con 0x7f8040105c70 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f8044b19700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803ffff700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f80401146f0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f802c00d750 tx=0x7f802c00da60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803dffb700 1 -- 172.21.15.90:0/867771505 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f802c004020 con 0x7f8040105c70 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803dffb700 1 -- 172.21.15.90:0/867771505 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f802c004830 con 0x7f8040105c70 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8040111d00 con 0x7f8040105c70 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f803dffb700 1 -- 172.21.15.90:0/867771505 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f802c005550 con 0x7f8040105c70 2024-04-06T14:09:23.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.428+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8040112260 con 0x7f8040105c70 2024-04-06T14:09:23.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.433+0000 7f803dffb700 1 -- 172.21.15.90:0/867771505 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f802c004ca0 con 0x7f8040105c70 2024-04-06T14:09:23.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.433+0000 7f8052f31b80 1 Processor -- start 2024-04-06T14:09:23.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f803dffb700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 0x7f80300a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f803dffb700 1 -- 172.21.15.90:0/867771505 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f802c0cc450 con 0x7f8040105c70 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f804531a700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 0x7f80300a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 -- start start 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa295b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a21d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa1ed990 con 0x5581fa2960a0 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa1eb350 con 0x5581fa29fde0 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f8052f31b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa1e6c40 con 0x5581fa295780 2024-04-06T14:09:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.434+0000 7f804531a700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 0x7f80300a37d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f8038003f10 tx=0x7f8038009100 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f803ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f803ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52872/0 (socket says 172.21.15.90:52872) 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f803ffff700 1 -- 172.21.15.90:0/2965756446 learned_addr learned my addr 172.21.15.90:0/2965756446 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a21d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 -- 172.21.15.90:0/2965756446 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 msgr2=0x5581fa295b60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa295b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 -- 172.21.15.90:0/2965756446 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 msgr2=0x5581fa29f8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 -- 172.21.15.90:0/2965756446 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5581fa0683b0 con 0x5581fa29fde0 2024-04-06T14:09:23.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f804531a700 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a21d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f8038009f30 tx=0x7f8038013000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c50674b6c19a7ad7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.435+0000 7f803ffff700 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.436+0000 7f802affd700 1 -- 172.21.15.90:0/2965756446 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8038007b90 con 0x5581fa29fde0 2024-04-06T14:09:23.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.436+0000 7f802affd700 1 -- 172.21.15.90:0/2965756446 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f803800f040 con 0x5581fa29fde0 2024-04-06T14:09:23.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.436+0000 7f802affd700 1 -- 172.21.15.90:0/2965756446 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f80380203f0 con 0x5581fa29fde0 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 -- 172.21.15.90:0/2965756446 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 msgr2=0x5581fa2a21d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a21d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f8038009f30 tx=0x7f8038013000 comp rx=0 tx=0).stop 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 -- 172.21.15.90:0/2965756446 shutdown_connections 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a21d0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa295b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2965756446 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29f8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 -- 172.21.15.90:0/2965756446 >> 172.21.15.90:0/2965756446 conn(0x5581fa1f46a0 msgr2=0x5581fa290a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.437+0000 7f8052f31b80 1 -- 172.21.15.90:0/2965756446 shutdown_connections 2024-04-06T14:09:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.438+0000 7f8052f31b80 1 -- 172.21.15.90:0/2965756446 wait complete. 2024-04-06T14:09:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.438+0000 7f8052f31b80 1 -- start start 2024-04-06T14:09:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.438+0000 7f8052f31b80 1 Processor -- start 2024-04-06T14:09:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa29ccf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa29d770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa182d80 con 0x5581fa2960a0 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa182a20 con 0x5581fa29fde0 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa06a6a0 con 0x5581fa295780 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.439+0000 7f8052f31b80 10 client.0 _ll_register_callbacks cb 0x5581fa18c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.440+0000 7f803ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.440+0000 7f803ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52888/0 (socket says 172.21.15.90:52888) 2024-04-06T14:09:23.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.440+0000 7f803ffff700 1 -- 172.21.15.90:0/3958633924 learned_addr learned my addr 172.21.15.90:0/3958633924 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.440+0000 7f803ffff700 1 -- 172.21.15.90:0/3958633924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 msgr2=0x5581fa29ccf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f803ffff700 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa29ccf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f803ffff700 1 -- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 msgr2=0x5581fa29d770 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f803ffff700 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa29d770 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f803ffff700 1 -- 172.21.15.90:0/3958633924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8040112260 con 0x5581fa2960a0 2024-04-06T14:09:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f803ffff700 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f802c000c00 tx=0x7f802c099d90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2fc98a70721a4a56 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.441+0000 7f801ffff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:23.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f802c0a19f0 con 0x5581fa2960a0 2024-04-06T14:09:23.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f802c0840c0 con 0x5581fa2960a0 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5581fa1a7380 con 0x5581fa2960a0 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f802c095c80 con 0x5581fa2960a0 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f8052f31b80 10 client.16095 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f80380042e0 con 0x5581fa2960a0 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.442+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5581fa1a7380 con 0x5581fa2960a0 2024-04-06T14:09:23.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f802c098a60 con 0x5581fa2960a0 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 10 client.16095 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 10 client.16095 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 20 client.16095 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa2afb00 0x5581fa2b1f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 4 client.16095 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5581fa1df700 con 0x5581fa2afb00 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8044b19700 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa2afb00 0x5581fa2b1f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f801ffff700 10 client.16095 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.443+0000 7f8044b19700 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa2afb00 0x5581fa2b1f20 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c8a4a9eb428fbd3a server_cookie=16d5e3c50d61ff05 in_seq=0 out_seq=0 2024-04-06T14:09:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f801ffff700 1 -- 172.21.15.90:0/3958633924 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5581fa1df700 con 0x5581fa2afb00 2024-04-06T14:09:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f801ffff700 10 client.16095 handle_command_reply: tid=0 2024-04-06T14:09:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 client.16095 shutdown 2024-04-06T14:09:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 msgr2=0x5581fa29d230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 secure :-1 s=READY pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f802c000c00 tx=0x7f802c099d90 comp rx=0 tx=0).stop 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 shutdown_connections 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa29d770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa2afb00 0x5581fa2b1f20 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa295780 0x5581fa29ccf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3958633924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2960a0 0x5581fa29d230 secure :-1 s=CLOSED pgs=540 cs=0 l=1 rev1=1 crypto rx=0x7f802c000c00 tx=0x7f802c099d90 comp rx=0 tx=0).stop 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 >> 172.21.15.90:0/3958633924 conn(0x5581fa1f46a0 msgr2=0x5581fa28fc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.446+0000 7f801ffff700 0 client.16095 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:23.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.447+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 shutdown_connections 2024-04-06T14:09:23.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.447+0000 7f8052f31b80 1 -- 172.21.15.90:0/3958633924 wait complete. 2024-04-06T14:09:23.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.447+0000 7f8052f31b80 20 client.16095 trim_cache size 0 max 16384 2024-04-06T14:09:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 Processor -- start 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 -- start start 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 0x5581fa290c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa29fde0 0x5581fa29c5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa296770 con 0x5581fa2a6a50 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa291650 con 0x5581fa295780 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.464+0000 7f8052f31b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa29db60 con 0x5581fa29fde0 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52892/0 (socket says 172.21.15.90:52892) 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 -- 172.21.15.90:0/2710401057 learned_addr learned my addr 172.21.15.90:0/2710401057 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f8044b19700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 0x5581fa290c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f803ffff700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa29fde0 0x5581fa29c5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 -- 172.21.15.90:0/2710401057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa29fde0 msgr2=0x5581fa29c5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa29fde0 0x5581fa29c5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 -- 172.21.15.90:0/2710401057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 msgr2=0x5581fa290c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 0x5581fa290c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 -- 172.21.15.90:0/2710401057 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5581fa014b60 con 0x5581fa2a6a50 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f8044b19700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 0x5581fa290c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f804531a700 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f80380045d0 tx=0x7f8038009170 comp rx=0 tx=0).ready entity=mon.0 client_cookie=24fccce50ded126f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f802b7fe700 1 -- 172.21.15.90:0/2710401057 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8038007b90 con 0x5581fa2a6a50 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f802b7fe700 1 -- 172.21.15.90:0/2710401057 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8038004da0 con 0x5581fa2a6a50 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.465+0000 7f802b7fe700 1 -- 172.21.15.90:0/2710401057 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8038005830 con 0x5581fa2a6a50 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- 172.21.15.90:0/2710401057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 msgr2=0x5581fa299f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 secure :-1 s=READY pgs=541 cs=0 l=1 rev1=1 crypto rx=0x7f80380045d0 tx=0x7f8038009170 comp rx=0 tx=0).stop 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- 172.21.15.90:0/2710401057 shutdown_connections 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa2a6a50 0x5581fa299f30 unknown :-1 s=CLOSED pgs=541 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa295780 0x5581fa290c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 --2- 172.21.15.90:0/2710401057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa29fde0 0x5581fa29c5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- 172.21.15.90:0/2710401057 >> 172.21.15.90:0/2710401057 conn(0x5581fa1f46a0 msgr2=0x5581fa290520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- 172.21.15.90:0/2710401057 shutdown_connections 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- 172.21.15.90:0/2710401057 wait complete. 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 -- start start 2024-04-06T14:09:23.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.466+0000 7f8052f31b80 1 Processor -- start 2024-04-06T14:09:23.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a4a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa2a6a50 0x5581fa2a50e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa2a57d0 con 0x5581fa295780 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa303940 con 0x5581fa29fde0 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581fa303a80 con 0x5581fa2a6a50 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8052f31b80 10 client.0 _ll_register_callbacks cb 0x5581fa18c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8044b19700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a4a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8044b19700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52896/0 (socket says 172.21.15.90:52896) 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8044b19700 1 -- 172.21.15.90:0/3619428338 learned_addr learned my addr 172.21.15.90:0/3619428338 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f804531a700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa2a6a50 0x5581fa2a50e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 -- 172.21.15.90:0/3619428338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa2a6a50 msgr2=0x5581fa2a50e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa2a6a50 0x5581fa2a50e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 -- 172.21.15.90:0/3619428338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 msgr2=0x5581fa2a64c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:23.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 -- 172.21.15.90:0/3619428338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8040114d60 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f8044b19700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f803ffff700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a4a40 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f802c004800 tx=0x7f802c0b90d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e5a904a6e0404956 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.467+0000 7f801ffff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f802c0a19f0 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f803800f040 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f802c0997d0 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f802c099990 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 10 client.25802 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5581fa0683b0 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f802c0a7020 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f802c098af0 con 0x5581fa29fde0 2024-04-06T14:09:23.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 10 client.25802 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 10 client.25802 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 20 client.25802 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa304dd0 0x5581fa3071c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 4 client.25802 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.468+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5581fa035cf0 con 0x5581fa304dd0 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f8044b19700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa304dd0 0x5581fa3071c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f801ffff700 10 client.25802 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f8044b19700 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa304dd0 0x5581fa3071c0 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=35675eacd5549f16 server_cookie=9967d8657bc2e6bd in_seq=0 out_seq=0 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f801ffff700 1 -- 172.21.15.90:0/3619428338 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5581fa035cf0 con 0x5581fa304dd0 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f801ffff700 10 client.25802 handle_command_reply: tid=0 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.469+0000 7f8052f31b80 1 client.25802 shutdown 2024-04-06T14:09:23.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 msgr2=0x5581fa2a4a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a4a40 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f802c004800 tx=0x7f802c0b90d0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 shutdown_connections 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581fa304dd0 0x5581fa3071c0 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581fa2a6a50 0x5581fa2a50e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5581fa295780 0x5581fa2a64c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 --2- 172.21.15.90:0/3619428338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581fa29fde0 0x5581fa2a4a40 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f801ffff700 0 client.25802 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 >> 172.21.15.90:0/3619428338 conn(0x5581fa1f46a0 msgr2=0x5581fa295000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 shutdown_connections 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 1 -- 172.21.15.90:0/3619428338 wait complete. 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8052f31b80 20 client.25802 trim_cache size 0 max 16384 2024-04-06T14:09:23.475 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:23.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 msgr2=0x7f80300a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 0x7f80300a37d0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f8038003f10 tx=0x7f8038009100 comp rx=0 tx=0).stop 2024-04-06T14:09:23.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 msgr2=0x7f80401146f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:23.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f80401146f0 secure :-1 s=READY pgs=539 cs=0 l=1 rev1=1 crypto rx=0x7f802c00d750 tx=0x7f802c00da60 comp rx=0 tx=0).stop 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 shutdown_connections 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f80300a1310 0x7f80300a37d0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f80401060e0 0x7f804010d340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8040103770 0x7f8040114170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 --2- 172.21.15.90:0/867771505 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8040105c70 0x7f80401146f0 unknown :-1 s=CLOSED pgs=539 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 >> 172.21.15.90:0/867771505 conn(0x7f80400769b0 msgr2=0x7f80401025f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 shutdown_connections 2024-04-06T14:09:23.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:23.470+0000 7f8046d7d700 1 -- 172.21.15.90:0/867771505 wait complete. 2024-04-06T14:09:23.790 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:23 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2962425947' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:23.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:23 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2962425947' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:23.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:23 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2962425947' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:24.484 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:09:24.484 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:09:24.484 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:09:24.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 -- 172.21.15.90:0/258707778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 msgr2=0x7f840c103bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 --2- 172.21.15.90:0/258707778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c103bd0 secure :-1 s=READY pgs=542 cs=0 l=1 rev1=1 crypto rx=0x7f8404009f90 tx=0x7f840400b330 comp rx=0 tx=0).stop 2024-04-06T14:09:24.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 -- 172.21.15.90:0/258707778 shutdown_connections 2024-04-06T14:09:24.695 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 --2- 172.21.15.90:0/258707778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c103bd0 unknown :-1 s=CLOSED pgs=542 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 --2- 172.21.15.90:0/258707778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 0x7f840c069310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 --2- 172.21.15.90:0/258707778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c069cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 -- 172.21.15.90:0/258707778 >> 172.21.15.90:0/258707778 conn(0x7f840c076db0 msgr2=0x7f840c0771c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.695+0000 7f8410b0d700 1 -- 172.21.15.90:0/258707778 shutdown_connections 2024-04-06T14:09:24.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.696+0000 7f8410b0d700 1 -- 172.21.15.90:0/258707778 wait complete. 2024-04-06T14:09:24.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.696+0000 7f8410b0d700 1 Processor -- start 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.696+0000 7f8410b0d700 1 -- start start 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c1a3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 0x7f840c19d640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f840c19dca0 con 0x7f840c103750 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f840c19de10 con 0x7f840c105c50 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8410b0d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f840c19dfd0 con 0x7f840c069850 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38892/0 (socket says 172.21.15.90:38892) 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 -- 172.21.15.90:0/3089886415 learned_addr learned my addr 172.21.15.90:0/3089886415 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840ad9d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 0x7f840c19d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f8409d9b700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c1a3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 -- 172.21.15.90:0/3089886415 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 msgr2=0x7f840c19d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 0x7f840c19d640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 -- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 msgr2=0x7f840c1a3a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c1a3a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.697+0000 7f840a59c700 1 -- 172.21.15.90:0/3089886415 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8404009c40 con 0x7f840c069850 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.698+0000 7f8409d9b700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c1a3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.698+0000 7f840a59c700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f840000b530 tx=0x7f840000b840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.698+0000 7f83fb7fe700 1 -- 172.21.15.90:0/3089886415 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8400011830 con 0x7f840c069850 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.698+0000 7f83fb7fe700 1 -- 172.21.15.90:0/3089886415 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8400011e50 con 0x7f840c069850 2024-04-06T14:09:24.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.698+0000 7f83fb7fe700 1 -- 172.21.15.90:0/3089886415 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f840000f2e0 con 0x7f840c069850 2024-04-06T14:09:24.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.699+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f840c19e2b0 con 0x7f840c069850 2024-04-06T14:09:24.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.699+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f840c106350 con 0x7f840c069850 2024-04-06T14:09:24.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.703+0000 7f83fb7fe700 1 -- 172.21.15.90:0/3089886415 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8400010380 con 0x7f840c069850 2024-04-06T14:09:24.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.704+0000 7f841ccc1b80 1 Processor -- start 2024-04-06T14:09:24.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.705+0000 7f841ccc1b80 1 -- start start 2024-04-06T14:09:24.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e97f800 0x564a5e97fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e989920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e8d7a10 con 0x564a5e989e60 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e8d53d0 con 0x564a5e97f800 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e8d0cc0 con 0x564a5e980120 2024-04-06T14:09:24.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f83fb7fe700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 0x7f83f40a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.706+0000 7f8409d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52932/0 (socket says 172.21.15.90:52932) 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 -- 172.21.15.90:0/3347464453 learned_addr learned my addr 172.21.15.90:0/3347464453 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f840a59c700 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e989920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f840ad9d700 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e97f800 0x564a5e97fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 -- 172.21.15.90:0/3347464453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 msgr2=0x564a5e989920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e989920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 -- 172.21.15.90:0/3347464453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e97f800 msgr2=0x564a5e97fbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e97f800 0x564a5e97fbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 -- 172.21.15.90:0/3347464453 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564a5e752430 con 0x564a5e989e60 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f8409d9b700 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f83fc009c70 tx=0x7f83fc009f80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80a29550e169aa42 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.707+0000 7f840ad9d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 0x7f83f40a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f83f8ff9700 1 -- 172.21.15.90:0/3347464453 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc004230 con 0x564a5e989e60 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f83fb7fe700 1 -- 172.21.15.90:0/3089886415 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f84000cc7a0 con 0x7f840c069850 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f83f8ff9700 1 -- 172.21.15.90:0/3347464453 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f83fc010e40 con 0x564a5e989e60 2024-04-06T14:09:24.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f83f8ff9700 1 -- 172.21.15.90:0/3347464453 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83fc012450 con 0x564a5e989e60 2024-04-06T14:09:24.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f841ccc1b80 1 -- 172.21.15.90:0/3347464453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 msgr2=0x564a5e98c250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f840ad9d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 0x7f83f40a37d0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f8404009550 tx=0x7f840400b330 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.708+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 secure :-1 s=READY pgs=543 cs=0 l=1 rev1=1 crypto rx=0x7f83fc009c70 tx=0x7f83fc009f80 comp rx=0 tx=0).stop 2024-04-06T14:09:24.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 -- 172.21.15.90:0/3347464453 shutdown_connections 2024-04-06T14:09:24.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e98c250 unknown :-1 s=CLOSED pgs=543 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e97f800 0x564a5e97fbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/3347464453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e989920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 -- 172.21.15.90:0/3347464453 >> 172.21.15.90:0/3347464453 conn(0x564a5e8de720 msgr2=0x564a5e97ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.709+0000 7f841ccc1b80 1 -- 172.21.15.90:0/3347464453 shutdown_connections 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.710+0000 7f841ccc1b80 1 -- 172.21.15.90:0/3347464453 wait complete. 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.710+0000 7f841ccc1b80 1 -- start start 2024-04-06T14:09:24.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.710+0000 7f841ccc1b80 1 Processor -- start 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e989880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e980120 0x564a5e986c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e86ce00 con 0x564a5e980120 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e86caa0 con 0x564a5e989e60 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.711+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e754720 con 0x564a5e97f800 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f841ccc1b80 10 client.0 _ll_register_callbacks cb 0x564a5e8764f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:56310/0 (socket says 172.21.15.90:56310) 2024-04-06T14:09:24.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 -- 172.21.15.90:0/2649623572 learned_addr learned my addr 172.21.15.90:0/2649623572 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 -- 172.21.15.90:0/2649623572 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 msgr2=0x564a5e989880 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e989880 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 -- 172.21.15.90:0/2649623572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e980120 msgr2=0x564a5e986c10 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e980120 0x564a5e986c10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 -- 172.21.15.90:0/2649623572 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83fc0098a0 con 0x564a5e989e60 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.712+0000 7f840ad9d700 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8404003e50 tx=0x7f8404008bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a4cdf773092c167b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.713+0000 7f83f1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.713+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8404025040 con 0x564a5e989e60 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.713+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f84040052c0 con 0x564a5e989e60 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.713+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564a5e891400 con 0x564a5e989e60 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.713+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f840400e040 con 0x564a5e989e60 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.714+0000 7f841ccc1b80 10 client.25811 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.714+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564a5e6febe0 con 0x564a5e989e60 2024-04-06T14:09:24.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.714+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564a5e891400 con 0x564a5e989e60 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f840402b860 con 0x564a5e989e60 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 10 client.25811 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 10 client.25811 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 20 client.25811 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5e999ae0 0x564a5e99bf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 4 client.25811 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:24.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.715+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564a5e8c9780 con 0x564a5e999ae0 2024-04-06T14:09:24.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.718+0000 7f840a59c700 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5e999ae0 0x564a5e99bf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.719+0000 7f83f1ffb700 10 client.25811 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.719+0000 7f840a59c700 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5e999ae0 0x564a5e99bf00 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=405238dcaade8dd0 server_cookie=8442b4f55de40107 in_seq=0 out_seq=0 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.719+0000 7f83f1ffb700 1 -- 172.21.15.90:0/2649623572 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564a5e8c9780 con 0x564a5e999ae0 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.719+0000 7f83f1ffb700 10 client.25811 handle_command_reply: tid=0 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 client.25811 shutdown 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 msgr2=0x564a5e987240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8404003e50 tx=0x7f8404008bc0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 shutdown_connections 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e987240 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5e999ae0 0x564a5e99bf00 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e989880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/2649623572 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e980120 0x564a5e986c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f83f1ffb700 0 client.25811 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:24.721 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.720+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 >> 172.21.15.90:0/2649623572 conn(0x564a5e8de720 msgr2=0x564a5e979ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.723+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 shutdown_connections 2024-04-06T14:09:24.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.723+0000 7f841ccc1b80 1 -- 172.21.15.90:0/2649623572 wait complete. 2024-04-06T14:09:24.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.723+0000 7f841ccc1b80 20 client.25811 trim_cache size 0 max 16384 2024-04-06T14:09:24.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.753+0000 7f841ccc1b80 1 Processor -- start 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.753+0000 7f841ccc1b80 1 -- start start 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e990ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e980120 0x564a5e995940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e97be40 con 0x564a5e989e60 2024-04-06T14:09:24.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e9812a0 con 0x564a5e980120 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e9723b0 con 0x564a5e97f800 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52948/0 (socket says 172.21.15.90:52948) 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 -- 172.21.15.90:0/1385331582 learned_addr learned my addr 172.21.15.90:0/1385331582 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840a59c700 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e990ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 -- 172.21.15.90:0/1385331582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 msgr2=0x564a5e990ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e990ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 -- 172.21.15.90:0/1385331582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e980120 msgr2=0x564a5e995940 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e980120 0x564a5e995940 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.754+0000 7f840ad9d700 1 -- 172.21.15.90:0/1385331582 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564a5e976be0 con 0x564a5e989e60 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.755+0000 7f840ad9d700 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f84040204d0 tx=0x7f8404009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3d17f449eaef3d5b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.755+0000 7f83f97fa700 1 -- 172.21.15.90:0/1385331582 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f840400e040 con 0x564a5e989e60 2024-04-06T14:09:24.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.755+0000 7f83f97fa700 1 -- 172.21.15.90:0/1385331582 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f84040205c0 con 0x564a5e989e60 2024-04-06T14:09:24.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.755+0000 7f83f97fa700 1 -- 172.21.15.90:0/1385331582 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8404005370 con 0x564a5e989e60 2024-04-06T14:09:24.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1385331582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 msgr2=0x564a5e97a420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 secure :-1 s=READY pgs=544 cs=0 l=1 rev1=1 crypto rx=0x7f84040204d0 tx=0x7f8404009f90 comp rx=0 tx=0).stop 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1385331582 shutdown_connections 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e989e60 0x564a5e97a420 unknown :-1 s=CLOSED pgs=544 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e97f800 0x564a5e990ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1385331582 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e980120 0x564a5e995940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1385331582 >> 172.21.15.90:0/1385331582 conn(0x564a5e8de720 msgr2=0x564a5e995230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1385331582 shutdown_connections 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1385331582 wait complete. 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 -- start start 2024-04-06T14:09:24.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.756+0000 7f841ccc1b80 1 Processor -- start 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e98fb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e9e7930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e9903f0 con 0x564a5e97f800 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e9e7e70 con 0x564a5e989e60 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564a5e9e8030 con 0x564a5e980120 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f841ccc1b80 10 client.0 _ll_register_callbacks cb 0x564a5e8764f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f840a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f840a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52960/0 (socket says 172.21.15.90:52960) 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.758+0000 7f840a59c700 1 -- 172.21.15.90:0/1783341645 learned_addr learned my addr 172.21.15.90:0/1783341645 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840ad9d700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e9e7930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 -- 172.21.15.90:0/1783341645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 msgr2=0x564a5e98fb60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e98fb60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 -- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 msgr2=0x564a5e9e7930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e9e7930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 -- 172.21.15.90:0/1783341645 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564a5e6febe0 con 0x564a5e97f800 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840ad9d700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e9e7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f840a59c700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f8400099690 tx=0x7f8400084780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f43e1ace61af37dc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f83f1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f84000a1e10 con 0x564a5e97f800 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8400084040 con 0x564a5e97f800 2024-04-06T14:09:24.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.759+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f83fc010e40 con 0x564a5e97f800 2024-04-06T14:09:24.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f84000b22b0 con 0x564a5e97f800 2024-04-06T14:09:24.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 10 client.16113 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564a5e896f20 con 0x564a5e97f800 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f84000b6080 con 0x564a5e97f800 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8400098e80 con 0x564a5e97f800 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 10 client.16113 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 10 client.16113 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 20 client.16113 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5ea00960 0x564a5ea02d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 4 client.16113 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.760+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564a5e71fd70 con 0x564a5ea00960 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.761+0000 7f8409d9b700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5ea00960 0x564a5ea02d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.761+0000 7f83f1ffb700 10 client.16113 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.761+0000 7f8409d9b700 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5ea00960 0x564a5ea02d50 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=66638a568891f1c6 server_cookie=1fb5ae5722bb8df9 in_seq=0 out_seq=0 2024-04-06T14:09:24.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.762+0000 7f83f1ffb700 1 -- 172.21.15.90:0/1783341645 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564a5e71fd70 con 0x564a5ea00960 2024-04-06T14:09:24.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.762+0000 7f83f1ffb700 10 client.16113 handle_command_reply: tid=0 2024-04-06T14:09:24.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.762+0000 7f841ccc1b80 1 client.16113 shutdown 2024-04-06T14:09:24.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 msgr2=0x564a5e98f620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 secure :-1 s=READY pgs=545 cs=0 l=1 rev1=1 crypto rx=0x7f8400099690 tx=0x7f8400084780 comp rx=0 tx=0).stop 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 shutdown_connections 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564a5e989e60 0x564a5e9e7930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564a5e97f800 0x564a5e98f620 unknown :-1 s=CLOSED pgs=545 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564a5ea00960 0x564a5ea02d50 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 --2- 172.21.15.90:0/1783341645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564a5e980120 0x564a5e98fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f83f1ffb700 0 client.16113 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 >> 172.21.15.90:0/1783341645 conn(0x564a5e8de720 msgr2=0x564a5e995ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 shutdown_connections 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.763+0000 7f841ccc1b80 1 -- 172.21.15.90:0/1783341645 wait complete. 2024-04-06T14:09:24.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.764+0000 7f841ccc1b80 20 client.16113 trim_cache size 0 max 16384 2024-04-06T14:09:24.765 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:24.765 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:09:24.765 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "6bfce7e8-ef9f-447d-93da-428d56b43913", 2024-04-06T14:09:24.765 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:09:24.765 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 msgr2=0x7f83f40a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 0x7f83f40a37d0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f8404009550 tx=0x7f840400b330 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 msgr2=0x7f840c1a3510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f840000b530 tx=0x7f840000b840 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 shutdown_connections 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83f40a1310 0x7f83f40a37d0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f840c105c50 0x7f840c19d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f840c069850 0x7f840c1a3510 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 --2- 172.21.15.90:0/3089886415 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f840c103750 0x7f840c1a3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 >> 172.21.15.90:0/3089886415 conn(0x7f840c076db0 msgr2=0x7f840c102730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:24.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 shutdown_connections 2024-04-06T14:09:24.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:24.765+0000 7f8410b0d700 1 -- 172.21.15.90:0/3089886415 wait complete. 2024-04-06T14:09:24.785 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:09:24.785 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:09:24.786 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:24 smithi090 ceph-mon[43420]: pgmap v123: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 19 MiB/s wr, 790 op/s 2024-04-06T14:09:24.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:24 smithi134 ceph-mon[47549]: pgmap v123: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 19 MiB/s wr, 790 op/s 2024-04-06T14:09:24.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:24 smithi105 ceph-mon[46969]: pgmap v123: 97 pgs: 97 active+clean; 5.8 GiB data, 8.3 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 19 MiB/s wr, 790 op/s 2024-04-06T14:09:25.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 -- 172.21.15.90:0/3656047304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87440ffe70 msgr2=0x7f8744100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 --2- 172.21.15.90:0/3656047304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87440ffe70 0x7f8744100250 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f8734009990 tx=0x7f8734009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 -- 172.21.15.90:0/3656047304 shutdown_connections 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 --2- 172.21.15.90:0/3656047304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8744104f70 0x7f874410caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 --2- 172.21.15.90:0/3656047304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f87440ffe70 0x7f8744100250 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 --2- 172.21.15.90:0/3656047304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8744100820 0x7f8744104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.025+0000 7f8748fb2700 1 -- 172.21.15.90:0/3656047304 >> 172.21.15.90:0/3656047304 conn(0x7f8744076d50 msgr2=0x7f8744077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.026+0000 7f8748fb2700 1 -- 172.21.15.90:0/3656047304 shutdown_connections 2024-04-06T14:09:25.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.026+0000 7f8748fb2700 1 -- 172.21.15.90:0/3656047304 wait complete. 2024-04-06T14:09:25.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 Processor -- start 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 -- start start 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87440ffe70 0x7f874410a460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8744104f70 0x7f874410a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f874419e260 con 0x7f87440ffe70 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f874419e3d0 con 0x7f874419d6a0 2024-04-06T14:09:25.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.027+0000 7f8748fb2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f874410e160 con 0x7f8744104f70 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.028+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.028+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:56344/0 (socket says 172.21.15.90:56344) 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.028+0000 7f8742d9d700 1 -- 172.21.15.90:0/1691237972 learned_addr learned my addr 172.21.15.90:0/1691237972 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 -- 172.21.15.90:0/1691237972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8744104f70 msgr2=0x7f874410a9a0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8744104f70 0x7f874410a9a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 -- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87440ffe70 msgr2=0x7f874410a460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87440ffe70 0x7f874410a460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 -- 172.21.15.90:0/1691237972 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f873c009c40 con 0x7f874419d6a0 2024-04-06T14:09:25.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.029+0000 7f8742d9d700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f873c00c370 tx=0x7f873c00c730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.030+0000 7f87337fe700 1 -- 172.21.15.90:0/1691237972 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873c017070 con 0x7f874419d6a0 2024-04-06T14:09:25.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.030+0000 7f87337fe700 1 -- 172.21.15.90:0/1691237972 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f873c00f040 con 0x7f874419d6a0 2024-04-06T14:09:25.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.030+0000 7f87337fe700 1 -- 172.21.15.90:0/1691237972 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873c00e050 con 0x7f874419d6a0 2024-04-06T14:09:25.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.031+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f87340095f0 con 0x7f874419d6a0 2024-04-06T14:09:25.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.031+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f874410e710 con 0x7f874419d6a0 2024-04-06T14:09:25.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.034+0000 7f8755166b80 1 Processor -- start 2024-04-06T14:09:25.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.035+0000 7f87337fe700 1 -- 172.21.15.90:0/1691237972 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f873c004500 con 0x7f874419d6a0 2024-04-06T14:09:25.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.036+0000 7f87337fe700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 0x7f872c0a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.036+0000 7f87337fe700 1 -- 172.21.15.90:0/1691237972 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f873c0cc0b0 con 0x7f874419d6a0 2024-04-06T14:09:25.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.036+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 0x7f872c0a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.036+0000 7f8755166b80 1 -- start start 2024-04-06T14:09:25.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.036+0000 7f8755166b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8755166b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb128a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8755166b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb151d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8755166b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfca60990 con 0x55acfcb12de0 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8755166b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfca5e350 con 0x55acfcb090a0 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8755166b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfca59c40 con 0x55acfcb08780 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38930/0 (socket says 172.21.15.90:38930) 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 -- 172.21.15.90:0/3079343522 learned_addr learned my addr 172.21.15.90:0/3079343522 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f8742d9d700 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb151d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 -- 172.21.15.90:0/3079343522 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 msgr2=0x55acfcb128a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb128a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 -- 172.21.15.90:0/3079343522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 msgr2=0x55acfcb151d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb151d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.037+0000 7f874259c700 1 -- 172.21.15.90:0/3079343522 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55acfc8db3b0 con 0x55acfcb08780 2024-04-06T14:09:25.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.038+0000 7f874259c700 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f8734006010 tx=0x7f8734005e80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=973e967abf1ad73d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.043+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 0x7f872c0a38a0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f8744106d30 tx=0x7f8738006cb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.052+0000 7f8730ff9700 1 -- 172.21.15.90:0/3079343522 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873401d070 con 0x55acfcb08780 2024-04-06T14:09:25.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.052+0000 7f8730ff9700 1 -- 172.21.15.90:0/3079343522 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f873400bc10 con 0x55acfcb08780 2024-04-06T14:09:25.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.052+0000 7f8730ff9700 1 -- 172.21.15.90:0/3079343522 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8734022c70 con 0x55acfcb08780 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 -- 172.21.15.90:0/3079343522 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 msgr2=0x55acfcb08b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f8734006010 tx=0x7f8734005e80 comp rx=0 tx=0).stop 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 -- 172.21.15.90:0/3079343522 shutdown_connections 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb151d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb08b60 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 --2- 172.21.15.90:0/3079343522 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb128a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.053+0000 7f8755166b80 1 -- 172.21.15.90:0/3079343522 >> 172.21.15.90:0/3079343522 conn(0x55acfca676a0 msgr2=0x55acfcb03a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.064+0000 7f8755166b80 1 -- 172.21.15.90:0/3079343522 shutdown_connections 2024-04-06T14:09:25.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.067+0000 7f8755166b80 1 -- 172.21.15.90:0/3079343522 wait complete. 2024-04-06T14:09:25.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.067+0000 7f8755166b80 1 -- start start 2024-04-06T14:09:25.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.067+0000 7f8755166b80 1 Processor -- start 2024-04-06T14:09:25.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb12890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb0fbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb10310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfc9f5d80 con 0x55acfcb12de0 2024-04-06T14:09:25.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfc9f5a20 con 0x55acfcb090a0 2024-04-06T14:09:25.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfc8dd6a0 con 0x55acfcb08780 2024-04-06T14:09:25.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8755166b80 10 client.0 _ll_register_callbacks cb 0x55acfc9ff470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f874259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb12890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb10310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb10310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52994/0 (socket says 172.21.15.90:52994) 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.068+0000 7f8742d9d700 1 -- 172.21.15.90:0/1623322748 learned_addr learned my addr 172.21.15.90:0/1623322748 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb0fbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 -- 172.21.15.90:0/1623322748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 msgr2=0x55acfcb12890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb12890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 -- 172.21.15.90:0/1623322748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 msgr2=0x55acfcb10310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb10310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 -- 172.21.15.90:0/1623322748 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8744068180 con 0x55acfcb090a0 2024-04-06T14:09:25.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f874259c700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb12890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8741d9b700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb0fbd0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f8744106dc0 tx=0x7f8738004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=99025f33f1698986 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8729ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873801a070 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55acfca1a380 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.069+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8738012a70 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.070+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f87380152e0 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.070+0000 7f8755166b80 10 client.25823 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.070+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55acfc887b60 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.070+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f873802b020 con 0x55acfcb090a0 2024-04-06T14:09:25.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.070+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f873801e410 con 0x55acfcb090a0 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 10 client.25823 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 10 client.25823 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 20 client.25823 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb229e0 0x55acfcb24e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 4 client.25823 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55acfca52700 con 0x55acfcb229e0 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.071+0000 7f874259c700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb229e0 0x55acfcb24e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.072+0000 7f874259c700 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb229e0 0x55acfcb24e00 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=332216303935f345 server_cookie=2b01abea49751f0d in_seq=0 out_seq=0 2024-04-06T14:09:25.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.073+0000 7f8729ffb700 10 client.25823 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.073+0000 7f8729ffb700 1 -- 172.21.15.90:0/1623322748 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55acfca52700 con 0x55acfcb229e0 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.073+0000 7f8729ffb700 10 client.25823 handle_command_reply: tid=0 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.073+0000 7f8755166b80 1 client.25823 shutdown 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 msgr2=0x55acfcb0fbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb0fbd0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f8744106dc0 tx=0x7f8738004750 comp rx=0 tx=0).stop 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 shutdown_connections 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb12de0 0x55acfcb10310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb229e0 0x55acfcb24e00 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb08780 0x55acfcb12890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 --2- 172.21.15.90:0/1623322748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb090a0 0x55acfcb0fbd0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8729ffb700 0 client.25823 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:25.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 >> 172.21.15.90:0/1623322748 conn(0x55acfca676a0 msgr2=0x55acfcb02c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 shutdown_connections 2024-04-06T14:09:25.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.074+0000 7f8755166b80 1 -- 172.21.15.90:0/1623322748 wait complete. 2024-04-06T14:09:25.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.075+0000 7f8755166b80 20 client.25823 trim_cache size 0 max 16384 2024-04-06T14:09:25.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.099+0000 7f8755166b80 1 Processor -- start 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.099+0000 7f8755166b80 1 -- start start 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb0fb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb19a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb05220 con 0x55acfcb08780 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb0a230 con 0x55acfcb12de0 2024-04-06T14:09:25.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8755166b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb0e090 con 0x55acfcb090a0 2024-04-06T14:09:25.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:56372/0 (socket says 172.21.15.90:56372) 2024-04-06T14:09:25.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 -- 172.21.15.90:0/42630434 learned_addr learned my addr 172.21.15.90:0/42630434 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8741d9b700 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb19a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 -- 172.21.15.90:0/42630434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 msgr2=0x55acfcb19a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb19a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 -- 172.21.15.90:0/42630434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 msgr2=0x55acfcb0fb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb0fb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.100+0000 7f8742d9d700 1 -- 172.21.15.90:0/42630434 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55acfca1fea0 con 0x55acfcb12de0 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.102+0000 7f8742d9d700 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f873c009690 tx=0x7f873c0990d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7ff9b82a476ec74d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.102+0000 7f87317fa700 1 -- 172.21.15.90:0/42630434 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873c09d000 con 0x55acfcb12de0 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.102+0000 7f87317fa700 1 -- 172.21.15.90:0/42630434 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f873c0af070 con 0x55acfcb12de0 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.102+0000 7f87317fa700 1 -- 172.21.15.90:0/42630434 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873c099430 con 0x55acfcb12de0 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 -- 172.21.15.90:0/42630434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 msgr2=0x55acfcb0ebb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f873c009690 tx=0x7f873c0990d0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 -- 172.21.15.90:0/42630434 shutdown_connections 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb0ebb0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb0fb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 --2- 172.21.15.90:0/42630434 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb19a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 -- 172.21.15.90:0/42630434 >> 172.21.15.90:0/42630434 conn(0x55acfca676a0 msgr2=0x55acfcb0e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 -- 172.21.15.90:0/42630434 shutdown_connections 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.103+0000 7f8755166b80 1 -- 172.21.15.90:0/42630434 wait complete. 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 -- start start 2024-04-06T14:09:25.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 Processor -- start 2024-04-06T14:09:25.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb18f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb76820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb76d60 con 0x55acfcb08780 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb76ed0 con 0x55acfcb12de0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55acfcb77090 con 0x55acfcb090a0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.104+0000 7f8755166b80 10 client.0 _ll_register_callbacks cb 0x55acfc9ff470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f8741d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb18f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:53004/0 (socket says 172.21.15.90:53004) 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 -- 172.21.15.90:0/2661611826 learned_addr learned my addr 172.21.15.90:0/2661611826 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f8742d9d700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb76820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 -- 172.21.15.90:0/2661611826 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 msgr2=0x55acfcb18f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb18f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 -- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 msgr2=0x55acfcb76820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:25.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb76820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 -- 172.21.15.90:0/2661611826 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f874410e710 con 0x55acfcb08780 2024-04-06T14:09:25.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f874259c700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f8734025040 tx=0x7f873400b380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=73e2af827f584256 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:25.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.105+0000 7f8729ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:25.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f873401d040 con 0x55acfcb08780 2024-04-06T14:09:25.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8734030cd0 con 0x55acfcb08780 2024-04-06T14:09:25.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f873400bc10 con 0x55acfcb08780 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8734030e90 con 0x55acfcb08780 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 10 client.16122 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55acfc8dfcd0 con 0x55acfcb08780 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f873403d020 con 0x55acfcb08780 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8734004a80 con 0x55acfcb08780 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 10 client.16122 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 10 client.16122 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 20 client.16122 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb78370 0x55acfcb7a760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 4 client.16122 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.106+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55acfc8a8cf0 con 0x55acfcb78370 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8741d9b700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb78370 0x55acfcb7a760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:25.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8729ffb700 10 client.16122 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8741d9b700 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb78370 0x55acfcb7a760 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cae46832075b150f server_cookie=396895989cc9476a in_seq=0 out_seq=0 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8729ffb700 1 -- 172.21.15.90:0/2661611826 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55acfc8a8cf0 con 0x55acfcb78370 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8729ffb700 10 client.16122 handle_command_reply: tid=0 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.107+0000 7f8755166b80 1 client.16122 shutdown 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 msgr2=0x55acfcb189d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 secure :-1 s=READY pgs=546 cs=0 l=1 rev1=1 crypto rx=0x7f8734025040 tx=0x7f873400b380 comp rx=0 tx=0).stop 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 shutdown_connections 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55acfcb12de0 0x55acfcb76820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55acfcb08780 0x55acfcb189d0 unknown :-1 s=CLOSED pgs=546 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55acfcb78370 0x55acfcb7a760 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 --2- 172.21.15.90:0/2661611826 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55acfcb090a0 0x55acfcb18f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 >> 172.21.15.90:0/2661611826 conn(0x55acfca676a0 msgr2=0x55acfcb0ed30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8729ffb700 0 client.16122 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:25.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 shutdown_connections 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.108+0000 7f8755166b80 1 -- 172.21.15.90:0/2661611826 wait complete. 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8755166b80 20 client.16122 trim_cache size 0 max 16384 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "6bfce7e8-ef9f-447d-93da-428d56b43913": { 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "6bfce7e8-ef9f-447d-93da-428d56b43913", 2024-04-06T14:09:25.111 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 msgr2=0x7f872c0a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 0x7f872c0a38a0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f8744106d30 tx=0x7f8738006cb0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 msgr2=0x7f874419db50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f873c00c370 tx=0x7f873c00c730 comp rx=0 tx=0).stop 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 shutdown_connections 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f872c0a13e0 0x7f872c0a38a0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f874419d6a0 0x7f874419db50 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f87440ffe70 0x7f874410a460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 --2- 172.21.15.90:0/1691237972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8744104f70 0x7f874410a9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.109+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 >> 172.21.15.90:0/1691237972 conn(0x7f8744076d50 msgr2=0x7f87440fdea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.110+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 shutdown_connections 2024-04-06T14:09:25.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:25.110+0000 7f8748fb2700 1 -- 172.21.15.90:0/1691237972 wait complete. 2024-04-06T14:09:25.122 INFO:tasks.cephfs.filesystem:scrub status for tag:6bfce7e8-ef9f-447d-93da-428d56b43913 - {'path': '/', 'tag': '6bfce7e8-ef9f-447d-93da-428d56b43913', 'options': 'recursive,force'} 2024-04-06T14:09:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:25 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:09:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:25 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:25 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:09:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:25 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:25.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:25 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:09:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:25 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:09:25.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:25 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:25 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:09:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:25 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:25.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:25 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:09:26.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:25 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:09:26.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:25 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:26.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:25 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:09:26.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:25 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:26.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:25 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:09:26.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:26 smithi134 ceph-mon[47549]: pgmap v124: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 26 MiB/s wr, 1.12k op/s 2024-04-06T14:09:26.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:26 smithi105 ceph-mon[46969]: pgmap v124: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 26 MiB/s wr, 1.12k op/s 2024-04-06T14:09:27.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:26 smithi090 ceph-mon[43420]: pgmap v124: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 26 MiB/s wr, 1.12k op/s 2024-04-06T14:09:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 220.0 s, 2556.1 tps, lat 39.101 ms stddev 45.465 2024-04-06T14:09:28.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:28 smithi134 ceph-mon[47549]: pgmap v125: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 15 MiB/s wr, 708 op/s 2024-04-06T14:09:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:28 smithi105 ceph-mon[46969]: pgmap v125: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 15 MiB/s wr, 708 op/s 2024-04-06T14:09:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:28 smithi090 ceph-mon[43420]: pgmap v125: 97 pgs: 97 active+clean; 5.9 GiB data, 8.4 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 15 MiB/s wr, 708 op/s 2024-04-06T14:09:30.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:29 smithi090 ceph-mon[43420]: pgmap v126: 97 pgs: 97 active+clean; 5.9 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 19 MiB/s wr, 831 op/s 2024-04-06T14:09:30.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:29 smithi134 ceph-mon[47549]: pgmap v126: 97 pgs: 97 active+clean; 5.9 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 19 MiB/s wr, 831 op/s 2024-04-06T14:09:30.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:29 smithi105 ceph-mon[46969]: pgmap v126: 97 pgs: 97 active+clean; 5.9 GiB data, 8.5 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 19 MiB/s wr, 831 op/s 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:30 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:30 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:30 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:31 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:09:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:31 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:09:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:31 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:31 smithi090 ceph-mon[43420]: pgmap v127: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 20 MiB/s wr, 751 op/s 2024-04-06T14:09:32.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:31 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:09:32.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:31 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:09:32.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:31 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:32.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:31 smithi134 ceph-mon[47549]: pgmap v127: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 20 MiB/s wr, 751 op/s 2024-04-06T14:09:32.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:31 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:09:32.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:31 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:09:32.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:31 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:09:32.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:31 smithi105 ceph-mon[46969]: pgmap v127: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 20 MiB/s wr, 751 op/s 2024-04-06T14:09:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 225.0 s, 2788.5 tps, lat 35.901 ms stddev 40.338 2024-04-06T14:09:34.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:33 smithi090 ceph-mon[43420]: pgmap v128: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 18 MiB/s wr, 671 op/s 2024-04-06T14:09:34.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:33 smithi134 ceph-mon[47549]: pgmap v128: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 18 MiB/s wr, 671 op/s 2024-04-06T14:09:34.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:33 smithi105 ceph-mon[46969]: pgmap v128: 97 pgs: 97 active+clean; 6.0 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 18 MiB/s wr, 671 op/s 2024-04-06T14:09:36.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:36 smithi105 ceph-mon[46969]: pgmap v129: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 28 MiB/s wr, 1.03k op/s 2024-04-06T14:09:37.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:36 smithi090 ceph-mon[43420]: pgmap v129: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 28 MiB/s wr, 1.03k op/s 2024-04-06T14:09:37.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:36 smithi134 ceph-mon[47549]: pgmap v129: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 28 MiB/s wr, 1.03k op/s 2024-04-06T14:09:38.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 230.0 s, 2608.6 tps, lat 37.382 ms stddev 43.835 2024-04-06T14:09:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:38 smithi134 ceph-mon[47549]: pgmap v130: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 21 MiB/s wr, 700 op/s 2024-04-06T14:09:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:38 smithi105 ceph-mon[46969]: pgmap v130: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 21 MiB/s wr, 700 op/s 2024-04-06T14:09:39.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:38 smithi090 ceph-mon[43420]: pgmap v130: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 21 MiB/s wr, 700 op/s 2024-04-06T14:09:40.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:40 smithi105 ceph-mon[46969]: pgmap v131: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 863 op/s 2024-04-06T14:09:41.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:40 smithi090 ceph-mon[43420]: pgmap v131: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 863 op/s 2024-04-06T14:09:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:40 smithi134 ceph-mon[47549]: pgmap v131: 97 pgs: 97 active+clean; 6.1 GiB data, 8.8 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 863 op/s 2024-04-06T14:09:42.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:41 smithi090 ceph-mon[43420]: pgmap v132: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 28 MiB/s wr, 1.07k op/s 2024-04-06T14:09:42.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:41 smithi134 ceph-mon[47549]: pgmap v132: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 28 MiB/s wr, 1.07k op/s 2024-04-06T14:09:42.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:41 smithi105 ceph-mon[46969]: pgmap v132: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 28 MiB/s wr, 1.07k op/s 2024-04-06T14:09:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 235.0 s, 224.8 tps, lat 302.215 ms stddev 180.510 2024-04-06T14:09:44.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:44 smithi105 ceph-mon[46969]: pgmap v133: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 21 MiB/s wr, 849 op/s 2024-04-06T14:09:45.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:44 smithi090 ceph-mon[43420]: pgmap v133: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 21 MiB/s wr, 849 op/s 2024-04-06T14:09:45.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:44 smithi134 ceph-mon[47549]: pgmap v133: 97 pgs: 97 active+clean; 6.2 GiB data, 8.9 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 21 MiB/s wr, 849 op/s 2024-04-06T14:09:46.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:46 smithi105 ceph-mon[46969]: pgmap v134: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 48 MiB/s wr, 2.32k op/s 2024-04-06T14:09:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:46 smithi090 ceph-mon[43420]: pgmap v134: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 48 MiB/s wr, 2.32k op/s 2024-04-06T14:09:47.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:46 smithi134 ceph-mon[47549]: pgmap v134: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 48 MiB/s wr, 2.32k op/s 2024-04-06T14:09:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 240.0 s, 3052.4 tps, lat 43.947 ms stddev 148.351 2024-04-06T14:09:48.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:48 smithi134 ceph-mon[47549]: pgmap v135: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 38 MiB/s wr, 1.96k op/s 2024-04-06T14:09:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:48 smithi105 ceph-mon[46969]: pgmap v135: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 38 MiB/s wr, 1.96k op/s 2024-04-06T14:09:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:48 smithi090 ceph-mon[43420]: pgmap v135: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 38 MiB/s wr, 1.96k op/s 2024-04-06T14:09:50.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:50 smithi105 ceph-mon[46969]: pgmap v136: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 40 MiB/s wr, 2.06k op/s 2024-04-06T14:09:51.037 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:50 smithi090 ceph-mon[43420]: pgmap v136: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 40 MiB/s wr, 2.06k op/s 2024-04-06T14:09:51.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:50 smithi134 ceph-mon[47549]: pgmap v136: 97 pgs: 97 active+clean; 6.4 GiB data, 9.1 GiB used, 1.0 TiB / 1.0 TiB avail; 120 KiB/s rd, 40 MiB/s wr, 2.06k op/s 2024-04-06T14:09:52.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:52 smithi105 ceph-mon[46969]: pgmap v137: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 43 MiB/s wr, 2.12k op/s 2024-04-06T14:09:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 245.0 s, 2896.2 tps, lat 34.477 ms stddev 38.180 2024-04-06T14:09:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:52 smithi090 ceph-mon[43420]: pgmap v137: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 43 MiB/s wr, 2.12k op/s 2024-04-06T14:09:53.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:52 smithi134 ceph-mon[47549]: pgmap v137: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 43 MiB/s wr, 2.12k op/s 2024-04-06T14:09:54.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:54 smithi105 ceph-mon[46969]: pgmap v138: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 113 KiB/s rd, 35 MiB/s wr, 1.78k op/s 2024-04-06T14:09:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:54 smithi090 ceph-mon[43420]: pgmap v138: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 113 KiB/s rd, 35 MiB/s wr, 1.78k op/s 2024-04-06T14:09:55.123 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:09:55.123 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:09:55.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:54 smithi134 ceph-mon[47549]: pgmap v138: 97 pgs: 97 active+clean; 6.4 GiB data, 9.2 GiB used, 1.0 TiB / 1.0 TiB avail; 113 KiB/s rd, 35 MiB/s wr, 1.78k op/s 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 -- 172.21.15.90:0/3660464300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178101920 msgr2=0x7f2178105960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 --2- 172.21.15.90:0/3660464300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178101920 0x7f2178105960 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f21680098f0 tx=0x7f2168009c00 comp rx=0 tx=0).stop 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 -- 172.21.15.90:0/3660464300 shutdown_connections 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 --2- 172.21.15.90:0/3660464300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2178106090 0x7f217810dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 --2- 172.21.15.90:0/3660464300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178100f70 0x7f2178101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 --2- 172.21.15.90:0/3660464300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178101920 0x7f2178105960 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 -- 172.21.15.90:0/3660464300 >> 172.21.15.90:0/3660464300 conn(0x7f21780fc7f0 msgr2=0x7f21780fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 -- 172.21.15.90:0/3660464300 shutdown_connections 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.296+0000 7f21803e6700 1 -- 172.21.15.90:0/3660464300 wait complete. 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 Processor -- start 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 -- start start 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 0x7f21781972d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 0x7f2178197810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 0x7f217819c030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f217819c630 con 0x7f2178106090 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f217819c7a0 con 0x7f217819bbb0 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f21803e6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f217819c960 con 0x7f2178100f70 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f217d981700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 0x7f2178197810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f217d981700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 0x7f2178197810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47428/0 (socket says 172.21.15.90:47428) 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f217d981700 1 -- 172.21.15.90:0/142245835 learned_addr learned my addr 172.21.15.90:0/142245835 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f217e983700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 0x7f217819c030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.297+0000 7f217e182700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 0x7f21781972d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 -- 172.21.15.90:0/142245835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 msgr2=0x7f21781972d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 0x7f21781972d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 -- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 msgr2=0x7f2178197810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 0x7f2178197810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 -- 172.21.15.90:0/142245835 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2168009520 con 0x7f217819bbb0 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e182700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 0x7f21781972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f217e983700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 0x7f217819c030 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f217000efc0 tx=0x7f217000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f216f7fe700 1 -- 172.21.15.90:0/142245835 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2170018070 con 0x7f217819bbb0 2024-04-06T14:09:55.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f216f7fe700 1 -- 172.21.15.90:0/142245835 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f217000f040 con 0x7f217819bbb0 2024-04-06T14:09:55.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.298+0000 7f216f7fe700 1 -- 172.21.15.90:0/142245835 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f217000ccc0 con 0x7f217819bbb0 2024-04-06T14:09:55.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.299+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2178072fe0 con 0x7f217819bbb0 2024-04-06T14:09:55.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.299+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2178073470 con 0x7f217819bbb0 2024-04-06T14:09:55.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.301+0000 7f216f7fe700 1 -- 172.21.15.90:0/142245835 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f2170003a00 con 0x7f217819bbb0 2024-04-06T14:09:55.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.302+0000 7f216f7fe700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 0x7f21640a7be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.302+0000 7f216f7fe700 1 -- 172.21.15.90:0/142245835 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f21700cbed0 con 0x7f217819bbb0 2024-04-06T14:09:55.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.303+0000 7f217e182700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 0x7f21640a7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.303+0000 7f217e182700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 0x7f21640a7be0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f2174011490 tx=0x7f2174009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.303+0000 7f218c59ab80 1 Processor -- start 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 -- start start 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79471eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d7947310b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d7947339e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d794676990 con 0x55d79471e780 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d794674350 con 0x55d7947315f0 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f218c59ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79466fc40 con 0x55d79471f0a0 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f217e182700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79471eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f217e182700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79471eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47434/0 (socket says 172.21.15.90:47434) 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f217e182700 1 -- 172.21.15.90:0/4257302192 learned_addr learned my addr 172.21.15.90:0/4257302192 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f217e983700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d7947339e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.304+0000 7f217d981700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d7947310b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 -- 172.21.15.90:0/4257302192 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 msgr2=0x55d7947310b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d7947310b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 -- 172.21.15.90:0/4257302192 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 msgr2=0x55d79471eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79471eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 -- 172.21.15.90:0/4257302192 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d7944f13b0 con 0x55d7947315f0 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217d981700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d7947310b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f217e983700 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d7947339e0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f21700c95f0 tx=0x7f2170098e00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=26d7b73e9d209eb5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f216cda5700 1 -- 172.21.15.90:0/4257302192 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f217001a2d0 con 0x55d7947315f0 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f216cda5700 1 -- 172.21.15.90:0/4257302192 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f21700996b0 con 0x55d7947315f0 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.305+0000 7f216cda5700 1 -- 172.21.15.90:0/4257302192 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21700a14c0 con 0x55d7947315f0 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- 172.21.15.90:0/4257302192 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 msgr2=0x55d7947339e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d7947339e0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f21700c95f0 tx=0x7f2170098e00 comp rx=0 tx=0).stop 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- 172.21.15.90:0/4257302192 shutdown_connections 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d7947339e0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79471eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 --2- 172.21.15.90:0/4257302192 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d7947310b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- 172.21.15.90:0/4257302192 >> 172.21.15.90:0/4257302192 conn(0x55d79467d6a0 msgr2=0x55d794719a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- 172.21.15.90:0/4257302192 shutdown_connections 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- 172.21.15.90:0/4257302192 wait complete. 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 -- start start 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.306+0000 7f218c59ab80 1 Processor -- start 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 0x55d79472e900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7947315f0 0x55d79472ee40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79460bd80 con 0x55d7947315f0 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79460ba20 con 0x55d79471f0a0 2024-04-06T14:09:55.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7944f36a0 con 0x55d79471e780 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f218c59ab80 10 client.0 _ll_register_callbacks cb 0x55d794615470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217e182700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217e182700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49198/0 (socket says 172.21.15.90:49198) 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217e182700 1 -- 172.21.15.90:0/2434605177 learned_addr learned my addr 172.21.15.90:0/2434605177 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217e983700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7947315f0 0x55d79472ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 0x55d79472e900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 -- 172.21.15.90:0/2434605177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 msgr2=0x55d79472e3c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 -- 172.21.15.90:0/2434605177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7947315f0 msgr2=0x55d79472ee40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7947315f0 0x55d79472ee40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.307+0000 7f217d981700 1 -- 172.21.15.90:0/2434605177 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2178073470 con 0x55d79471f0a0 2024-04-06T14:09:55.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f217e182700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f217d981700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 0x55d79472e900 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f2168009fd0 tx=0x7f216800f690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f7740b53e95b123e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f2161ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f216801c070 con 0x55d79471f0a0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f216800b710 con 0x55d79471f0a0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d794630380 con 0x55d79471f0a0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f216800be90 con 0x55d79471f0a0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f218c59ab80 10 client.25841 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.308+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d79449db60 con 0x55d79471f0a0 2024-04-06T14:09:55.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d794630380 con 0x55d79471f0a0 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2168015020 con 0x55d79471f0a0 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 10 client.25841 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 10 client.25841 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 20 client.25841 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d794741250 0x55d794743670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 4 client.25841 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d794668700 con 0x55d794741250 2024-04-06T14:09:55.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.309+0000 7f217e182700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d794741250 0x55d794743670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f2161ffb700 10 client.25841 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f217e182700 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d794741250 0x55d794743670 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=df8e0695cc701f29 server_cookie=531cea02521c4813 in_seq=0 out_seq=0 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f2161ffb700 1 -- 172.21.15.90:0/2434605177 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d794668700 con 0x55d794741250 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f2161ffb700 10 client.25841 handle_command_reply: tid=0 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f218c59ab80 1 client.25841 shutdown 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 msgr2=0x55d79472e900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.310+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 0x55d79472e900 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f2168009fd0 tx=0x7f216800f690 comp rx=0 tx=0).stop 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 shutdown_connections 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7947315f0 0x55d79472ee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d794741250 0x55d794743670 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471e780 0x55d79472e3c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2434605177 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471f0a0 0x55d79472e900 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 >> 172.21.15.90:0/2434605177 conn(0x55d79467d6a0 msgr2=0x55d794718c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f2161ffb700 0 client.25841 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:55.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 shutdown_connections 2024-04-06T14:09:55.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 1 -- 172.21.15.90:0/2434605177 wait complete. 2024-04-06T14:09:55.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.311+0000 7f218c59ab80 20 client.25841 trim_cache size 0 max 16384 2024-04-06T14:09:55.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.332+0000 7f218c59ab80 1 Processor -- start 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.332+0000 7f218c59ab80 1 -- start start 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79472d560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d794738270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79471fa10 con 0x55d79471e780 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7947292a0 con 0x55d7947315f0 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f218c59ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79471eeb0 con 0x55d79471f0a0 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e983700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217d981700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d794738270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79472d560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217d981700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d794738270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49206/0 (socket says 172.21.15.90:49206) 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217d981700 1 -- 172.21.15.90:0/1350182414 learned_addr learned my addr 172.21.15.90:0/1350182414 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e983700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52122/0 (socket says 172.21.15.90:52122) 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 -- 172.21.15.90:0/1350182414 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 msgr2=0x55d794738270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d794738270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 -- 172.21.15.90:0/1350182414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 msgr2=0x55d79471de80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 -- 172.21.15.90:0/1350182414 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d794635ea0 con 0x55d79471e780 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e983700 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.333+0000 7f217e182700 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79472d560 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x55d794743f50 tx=0x7f2174014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f12e83f884ff1f8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f216d5a6700 1 -- 172.21.15.90:0/1350182414 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2174024040 con 0x55d79471e780 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f216d5a6700 1 -- 172.21.15.90:0/1350182414 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2174029dc0 con 0x55d79471e780 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f216d5a6700 1 -- 172.21.15.90:0/1350182414 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f217401eac0 con 0x55d79471e780 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 -- 172.21.15.90:0/1350182414 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 msgr2=0x55d79472d560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79472d560 secure :-1 s=READY pgs=547 cs=0 l=1 rev1=1 crypto rx=0x55d794743f50 tx=0x7f2174014000 comp rx=0 tx=0).stop 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 -- 172.21.15.90:0/1350182414 shutdown_connections 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7947315f0 0x55d79471de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471e780 0x55d79472d560 unknown :-1 s=CLOSED pgs=547 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 --2- 172.21.15.90:0/1350182414 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d79471f0a0 0x55d794738270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 -- 172.21.15.90:0/1350182414 >> 172.21.15.90:0/1350182414 conn(0x55d79467d6a0 msgr2=0x55d79471d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.334+0000 7f218c59ab80 1 -- 172.21.15.90:0/1350182414 shutdown_connections 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 -- 172.21.15.90:0/1350182414 wait complete. 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 -- start start 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 Processor -- start 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 0x55d794738040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 0x55d794736bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7947372d0 con 0x55d79471f0a0 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d794737440 con 0x55d79471e780 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d79478f060 con 0x55d7947315f0 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f218c59ab80 10 client.0 _ll_register_callbacks cb 0x55d794615470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:55.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f217d981700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f217e182700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 0x55d794738040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f217d981700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47460/0 (socket says 172.21.15.90:47460) 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f217d981700 1 -- 172.21.15.90:0/2998927672 learned_addr learned my addr 172.21.15.90:0/2998927672 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.335+0000 7f217e983700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 0x55d794736bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 -- 172.21.15.90:0/2998927672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 msgr2=0x55d794736bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 0x55d794736bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 -- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 msgr2=0x55d794738040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 0x55d794738040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 -- 172.21.15.90:0/2998927672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d7944f13b0 con 0x55d79471f0a0 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217e983700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 0x55d794736bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217e182700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 0x55d794738040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:55.342 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f217d981700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f2168009fd0 tx=0x7f2168003db0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dbafc6e27399188 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f216801c040 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f21680058e0 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f21700996b0 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f216800be90 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f218c59ab80 10 client.16140 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f2174017070 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2168041020 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2168015070 con 0x55d79471f0a0 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f218c59ab80 10 client.16140 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.336+0000 7f218c59ab80 10 client.16140 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.337+0000 7f218c59ab80 20 client.16140 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:55.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.337+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7947902b0 0x55d7947287a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.337+0000 7f218c59ab80 4 client.16140 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:09:55.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.337+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d7944becf0 con 0x55d7947902b0 2024-04-06T14:09:55.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.337+0000 7f217e182700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7947902b0 0x55d7947287a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f2161ffb700 10 client.16140 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f217e182700 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7947902b0 0x55d7947287a0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d558e36e28167320 server_cookie=891c32508fe814f4 in_seq=0 out_seq=0 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f2161ffb700 1 -- 172.21.15.90:0/2998927672 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55d7944becf0 con 0x55d7947902b0 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f2161ffb700 10 client.16140 handle_command_reply: tid=0 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f218c59ab80 1 client.16140 shutdown 2024-04-06T14:09:55.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 msgr2=0x55d794736680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.338+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 secure :-1 s=READY pgs=548 cs=0 l=1 rev1=1 crypto rx=0x7f2168009fd0 tx=0x7f2168003db0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 shutdown_connections 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7947902b0 0x55d7947287a0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7947315f0 0x55d794736bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f2161ffb700 0 client.16140 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d79471e780 0x55d794738040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 --2- 172.21.15.90:0/2998927672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d79471f0a0 0x55d794736680 unknown :-1 s=CLOSED pgs=548 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 >> 172.21.15.90:0/2998927672 conn(0x55d79467d6a0 msgr2=0x55d79472d3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 shutdown_connections 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.339+0000 7f218c59ab80 1 -- 172.21.15.90:0/2998927672 wait complete. 2024-04-06T14:09:55.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f218c59ab80 20 client.16140 trim_cache size 0 max 16384 2024-04-06T14:09:55.347 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:55.347 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:09:55.347 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:09:55.347 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 msgr2=0x7f21640a7be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 0x7f21640a7be0 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f2174011490 tx=0x7f2174009260 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 msgr2=0x7f217819c030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 0x7f217819c030 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f217000efc0 tx=0x7f217000c3a0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 shutdown_connections 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f21640a5720 0x7f21640a7be0 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217819bbb0 0x7f217819c030 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2178100f70 0x7f21781972d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 --2- 172.21.15.90:0/142245835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2178106090 0x7f2178197810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 >> 172.21.15.90:0/142245835 conn(0x7f21780fc7f0 msgr2=0x7f21780fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 shutdown_connections 2024-04-06T14:09:55.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.340+0000 7f21803e6700 1 -- 172.21.15.90:0/142245835 wait complete. 2024-04-06T14:09:55.356 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:09:55.356 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:09:55.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 -- 172.21.15.90:0/3585528458 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 msgr2=0x7f3a3010c9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 --2- 172.21.15.90:0/3585528458 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a3010c9b0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f3a200099c0 tx=0x7f3a20009cd0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 -- 172.21.15.90:0/3585528458 shutdown_connections 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 --2- 172.21.15.90:0/3585528458 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a3010c9b0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 --2- 172.21.15.90:0/3585528458 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a30105fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 --2- 172.21.15.90:0/3585528458 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a300fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 -- 172.21.15.90:0/3585528458 >> 172.21.15.90:0/3585528458 conn(0x7f3a300fb5e0 msgr2=0x7f3a300fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 -- 172.21.15.90:0/3585528458 shutdown_connections 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.537+0000 7f3a35877700 1 -- 172.21.15.90:0/3585528458 wait complete. 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.538+0000 7f3a35877700 1 Processor -- start 2024-04-06T14:09:55.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.538+0000 7f3a35877700 1 -- start start 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a301991e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a30199720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a3019dac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a3019e100 con 0x7f3a300ffb30 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a3019e270 con 0x7f3a301004f0 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a35877700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a3019e430 con 0x7f3a30105bf0 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a2effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a301991e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a2effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a301991e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47478/0 (socket says 172.21.15.90:47478) 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a2effd700 1 -- 172.21.15.90:0/2556573788 learned_addr learned my addr 172.21.15.90:0/2556573788 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:55.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a2f7fe700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a3019dac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.539+0000 7f3a2e7fc700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a30199720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 -- 172.21.15.90:0/2556573788 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 msgr2=0x7f3a30199720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a30199720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 -- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 msgr2=0x7f3a301991e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a301991e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 -- 172.21.15.90:0/2556573788 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a200095f0 con 0x7f3a30105bf0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a2f7fe700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a3019dac0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f3a200098e0 tx=0x7f3a2000f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a2001c070 con 0x7f3a30105bf0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a3019e6b0 con 0x7f3a30105bf0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3a2000fdb0 con 0x7f3a30105bf0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a20022c90 con 0x7f3a30105bf0 2024-04-06T14:09:55.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.540+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a301a5f50 con 0x7f3a30105bf0 2024-04-06T14:09:55.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.541+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a30107fd0 con 0x7f3a30105bf0 2024-04-06T14:09:55.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.542+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3a20020070 con 0x7f3a30105bf0 2024-04-06T14:09:55.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.543+0000 7f3a34875700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 0x7f3a180a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:55.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.543+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f3a200ce2c0 con 0x7f3a30105bf0 2024-04-06T14:09:55.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.545+0000 7f3a2effd700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 0x7f3a180a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:55.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.545+0000 7f3a2effd700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 0x7f3a180a3750 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f3a28007fd0 tx=0x7f3a2800c040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:55.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.547+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3a20097770 con 0x7f3a30105bf0 2024-04-06T14:09:55.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.899+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3a301011e0 con 0x7f3a30105bf0 2024-04-06T14:09:55.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.900+0000 7f3a34875700 1 -- 172.21.15.90:0/2556573788 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f3a2009ad90 con 0x7f3a30105bf0 2024-04-06T14:09:55.900 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:09:55.900 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:09:55.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.902+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 msgr2=0x7f3a180a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 0x7f3a180a3750 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f3a28007fd0 tx=0x7f3a2800c040 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 msgr2=0x7f3a3019dac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a3019dac0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f3a200098e0 tx=0x7f3a2000f690 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 shutdown_connections 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3a180a1290 0x7f3a180a3750 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3a30105bf0 0x7f3a3019dac0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3a300ffb30 0x7f3a301991e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 --2- 172.21.15.90:0/2556573788 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3a301004f0 0x7f3a30199720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 >> 172.21.15.90:0/2556573788 conn(0x7f3a300fb5e0 msgr2=0x7f3a300fcda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 shutdown_connections 2024-04-06T14:09:55.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:55.903+0000 7f3a35877700 1 -- 172.21.15.90:0/2556573788 wait complete. 2024-04-06T14:09:55.905 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:09:55.917 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:55.917 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:09:56.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 -- 172.21.15.90:0/1203799979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010b720 msgr2=0x7f8a5010bba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 --2- 172.21.15.90:0/1203799979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010b720 0x7f8a5010bba0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f8a38009990 tx=0x7f8a38009ca0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 -- 172.21.15.90:0/1203799979 shutdown_connections 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 --2- 172.21.15.90:0/1203799979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010c0e0 0x7f8a501185e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 --2- 172.21.15.90:0/1203799979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a50111b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 --2- 172.21.15.90:0/1203799979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010b720 0x7f8a5010bba0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.104+0000 7f8a5657a700 1 -- 172.21.15.90:0/1203799979 >> 172.21.15.90:0/1203799979 conn(0x7f8a5006bd10 msgr2=0x7f8a50109dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.105+0000 7f8a5657a700 1 -- 172.21.15.90:0/1203799979 shutdown_connections 2024-04-06T14:09:56.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.105+0000 7f8a5657a700 1 -- 172.21.15.90:0/1203799979 wait complete. 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.105+0000 7f8a5657a700 1 Processor -- start 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 -- start start 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 0x7f8a5006c6b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010c0e0 0x7f8a5006cbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a5006d130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a501b2260 con 0x7f8a5010b720 2024-04-06T14:09:56.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a501b23a0 con 0x7f8a5010c0e0 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a5657a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a501b2510 con 0x7f8a50111720 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 0x7f8a5006c6b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 0x7f8a5006c6b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47492/0 (socket says 172.21.15.90:47492) 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a4ffff700 1 -- 172.21.15.90:0/25885550 learned_addr learned my addr 172.21.15.90:0/25885550 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a54b17700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a5006d130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.106+0000 7f8a47fff700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010c0e0 0x7f8a5006cbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 -- 172.21.15.90:0/25885550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010c0e0 msgr2=0x7f8a5006cbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010c0e0 0x7f8a5006cbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 -- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 msgr2=0x7f8a5006c6b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 0x7f8a5006c6b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 -- 172.21.15.90:0/25885550 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a380095f0 con 0x7f8a50111720 2024-04-06T14:09:56.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a54b17700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a5006d130 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f8a4800be20 tx=0x7f8a4800be50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a4dffb700 1 -- 172.21.15.90:0/25885550 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a4800cad0 con 0x7f8a50111720 2024-04-06T14:09:56.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a4dffb700 1 -- 172.21.15.90:0/25885550 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a4800cc30 con 0x7f8a50111720 2024-04-06T14:09:56.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a4dffb700 1 -- 172.21.15.90:0/25885550 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a480125a0 con 0x7f8a50111720 2024-04-06T14:09:56.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a501b27f0 con 0x7f8a50111720 2024-04-06T14:09:56.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.107+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a501b2cb0 con 0x7f8a50111720 2024-04-06T14:09:56.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.110+0000 7f8a4dffb700 1 -- 172.21.15.90:0/25885550 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8a48014440 con 0x7f8a50111720 2024-04-06T14:09:56.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.111+0000 7f8a4dffb700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 0x7f8a3c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.111+0000 7f8a4dffb700 1 -- 172.21.15.90:0/25885550 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a48006e40 con 0x7f8a50111720 2024-04-06T14:09:56.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.111+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 0x7f8a3c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 Processor -- start 2024-04-06T14:09:56.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 -- start start 2024-04-06T14:09:56.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98ebb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98f81d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b9843990 con 0x5560b98ec0a0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b9841350 con 0x5560b98eb780 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b983cc40 con 0x5560b98f5de0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 0x7f8a3c0a37d0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f8a40005950 tx=0x7f8a4000b500 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a47fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a47fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47500/0 (socket says 172.21.15.90:47500) 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a54b17700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98f81d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a47fff700 1 -- 172.21.15.90:0/3404063265 learned_addr learned my addr 172.21.15.90:0/3404063265 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.112+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98ebb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 -- 172.21.15.90:0/3404063265 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 msgr2=0x5560b98ebb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98ebb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 -- 172.21.15.90:0/3404063265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 msgr2=0x5560b98f58a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 -- 172.21.15.90:0/3404063265 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5560b96be3b0 con 0x5560b98f5de0 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a47fff700 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a54b17700 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98f81d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f8a480c9c70 tx=0x7f8a48004ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=be2204965e312739 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a467fc700 1 -- 172.21.15.90:0/3404063265 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a48099620 con 0x5560b98f5de0 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a467fc700 1 -- 172.21.15.90:0/3404063265 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a4809d450 con 0x5560b98f5de0 2024-04-06T14:09:56.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.113+0000 7f8a467fc700 1 -- 172.21.15.90:0/3404063265 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a48099880 con 0x5560b98f5de0 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 -- 172.21.15.90:0/3404063265 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 msgr2=0x5560b98f81d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98f81d0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f8a480c9c70 tx=0x7f8a48004ad0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 -- 172.21.15.90:0/3404063265 shutdown_connections 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98f81d0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98ebb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/3404063265 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98f58a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.114+0000 7f8a6272eb80 1 -- 172.21.15.90:0/3404063265 >> 172.21.15.90:0/3404063265 conn(0x5560b984a6a0 msgr2=0x5560b98e6a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.115+0000 7f8a6272eb80 1 -- 172.21.15.90:0/3404063265 shutdown_connections 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.115+0000 7f8a6272eb80 1 -- 172.21.15.90:0/3404063265 wait complete. 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.115+0000 7f8a6272eb80 1 -- start start 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.115+0000 7f8a6272eb80 1 Processor -- start 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 0x5560b98f2b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98f5de0 0x5560b98f3240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b97d8d80 con 0x5560b98f5de0 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b97d8a20 con 0x5560b98ec0a0 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b96c06a0 con 0x5560b98eb780 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a6272eb80 10 client.0 _ll_register_callbacks cb 0x5560b97e2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49262/0 (socket says 172.21.15.90:49262) 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 -- 172.21.15.90:0/166986605 learned_addr learned my addr 172.21.15.90:0/166986605 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a47fff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 0x5560b98f2b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 -- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 msgr2=0x5560b98f2b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 0x5560b98f2b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 -- 172.21.15.90:0/166986605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98f5de0 msgr2=0x5560b98f3240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98f5de0 0x5560b98f3240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 -- 172.21.15.90:0/166986605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a50118ee0 con 0x5560b98eb780 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a47fff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 0x5560b98f2b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f8a400116f0 tx=0x7f8a400047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=39a9a06a781f9cbb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a2f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.116+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a40004440 con 0x5560b98eb780 2024-04-06T14:09:56.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.117+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a40017070 con 0x5560b98eb780 2024-04-06T14:09:56.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.117+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5560b97fd380 con 0x5560b98eb780 2024-04-06T14:09:56.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a400045a0 con 0x5560b98eb780 2024-04-06T14:09:56.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 10 client.25768 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f8a501b2cb0 con 0x5560b98eb780 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5560b97fd380 con 0x5560b98eb780 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a4001e860 con 0x5560b98eb780 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 10 client.25768 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 10 client.25768 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 20 client.25768 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b9916bb0 0x5560b9918fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 4 client.25768 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.118+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5560b9835700 con 0x5560b9916bb0 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.119+0000 7f8a47fff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b9916bb0 0x5560b9918fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.119+0000 7f8a2f7fe700 10 client.25768 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.119+0000 7f8a47fff700 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b9916bb0 0x5560b9918fd0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=435de9c587c6ef4a server_cookie=d35a397e4383c35a in_seq=0 out_seq=0 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/166986605 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5560b9835700 con 0x5560b9916bb0 2024-04-06T14:09:56.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a2f7fe700 10 client.25768 handle_command_reply: tid=0 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 client.25768 shutdown 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 msgr2=0x5560b98f5860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f8a400116f0 tx=0x7f8a400047c0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 shutdown_connections 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b9916bb0 0x5560b9918fd0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98f5de0 0x5560b98f3240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98f5860 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a2f7fe700 0 client.25768 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/166986605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98ec0a0 0x5560b98f2b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.120+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 >> 172.21.15.90:0/166986605 conn(0x5560b984a6a0 msgr2=0x5560b98e5c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.121+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 shutdown_connections 2024-04-06T14:09:56.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.121+0000 7f8a6272eb80 1 -- 172.21.15.90:0/166986605 wait complete. 2024-04-06T14:09:56.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.122+0000 7f8a6272eb80 20 client.25768 trim_cache size 0 max 16384 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.137+0000 7f8a6272eb80 1 Processor -- start 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 -- start start 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98ede40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98ee380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b98e77f0 con 0x5560b98ec0a0 2024-04-06T14:09:56.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b98eca10 con 0x5560b98eb780 2024-04-06T14:09:56.139 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b98fcfa0 con 0x5560b98f5de0 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a54b17700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98ee380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a47fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98ede40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a4ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52202/0 (socket says 172.21.15.90:52202) 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a54b17700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98ee380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49276/0 (socket says 172.21.15.90:49276) 2024-04-06T14:09:56.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.138+0000 7f8a4ffff700 1 -- 172.21.15.90:0/2066027670 learned_addr learned my addr 172.21.15.90:0/2066027670 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 -- 172.21.15.90:0/2066027670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 msgr2=0x5560b98ee380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98ee380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 -- 172.21.15.90:0/2066027670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 msgr2=0x5560b98ede40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98ede40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 -- 172.21.15.90:0/2066027670 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5560b966ab60 con 0x5560b98eb780 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f8a400116c0 tx=0x7f8a400283f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d046108f8ebd9e1b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a46ffd700 1 -- 172.21.15.90:0/2066027670 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a40004440 con 0x5560b98eb780 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a46ffd700 1 -- 172.21.15.90:0/2066027670 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a40037070 con 0x5560b98eb780 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.139+0000 7f8a46ffd700 1 -- 172.21.15.90:0/2066027670 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a40003bc0 con 0x5560b98eb780 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 -- 172.21.15.90:0/2066027670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 msgr2=0x5560b98fca60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f8a400116c0 tx=0x7f8a400283f0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 -- 172.21.15.90:0/2066027670 shutdown_connections 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98f5de0 0x5560b98ee380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98eb780 0x5560b98fca60 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/2066027670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98ede40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.140+0000 7f8a6272eb80 1 -- 172.21.15.90:0/2066027670 >> 172.21.15.90:0/2066027670 conn(0x5560b984a6a0 msgr2=0x5560b98f2090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.142+0000 7f8a6272eb80 1 -- 172.21.15.90:0/2066027670 shutdown_connections 2024-04-06T14:09:56.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.142+0000 7f8a6272eb80 1 -- 172.21.15.90:0/2066027670 wait complete. 2024-04-06T14:09:56.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.142+0000 7f8a6272eb80 1 -- start start 2024-04-06T14:09:56.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 Processor -- start 2024-04-06T14:09:56.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98fc9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98fb060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98f5de0 0x5560b98fb6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b98fbdc0 con 0x5560b98ec0a0 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b98fbf30 con 0x5560b98f5de0 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5560b996a9e0 con 0x5560b98eb780 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a6272eb80 10 client.0 _ll_register_callbacks cb 0x7f8a40010e40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a47fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98fb060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a47fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98fb060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47522/0 (socket says 172.21.15.90:47522) 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a47fff700 1 -- 172.21.15.90:0/1547416280 learned_addr learned my addr 172.21.15.90:0/1547416280 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.143+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98fc9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.144+0000 7f8a4ffff700 1 -- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98f5de0 msgr2=0x5560b98fb6e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.144+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98f5de0 0x5560b98fb6e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.144+0000 7f8a4ffff700 1 -- 172.21.15.90:0/1547416280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 msgr2=0x5560b98fb060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.144+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98fb060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.144+0000 7f8a4ffff700 1 -- 172.21.15.90:0/1547416280 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a501b2cb0 con 0x5560b98eb780 2024-04-06T14:09:56.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.145+0000 7f8a4ffff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98fc9a0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8a40028460 tx=0x7f8a400057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=eff0e3e32042d0df server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.145+0000 7f8a2f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:09:56.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.145+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a4001e830 con 0x5560b98eb780 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.145+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8a40017070 con 0x5560b98eb780 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.145+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8a4809d450 con 0x5560b98eb780 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a4001e990 con 0x5560b98eb780 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 10 client.25774 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5560b9802ea0 con 0x5560b98eb780 2024-04-06T14:09:56.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a4809d450 con 0x5560b98eb780 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a40028a80 con 0x5560b98eb780 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 10 client.25774 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 10 client.25774 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 20 client.25774 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b996bd50 0x5560b996e170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 4 client.25774 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5560b968bcf0 con 0x5560b996bd50 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.146+0000 7f8a47fff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b996bd50 0x5560b996e170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.147+0000 7f8a47fff700 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b996bd50 0x5560b996e170 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=9430cea94fdedd0c server_cookie=52d28dfeaf87a29e in_seq=0 out_seq=0 2024-04-06T14:09:56.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.147+0000 7f8a2f7fe700 10 client.25774 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:56.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a2f7fe700 1 -- 172.21.15.90:0/1547416280 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5560b968bcf0 con 0x5560b996bd50 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a2f7fe700 10 client.25774 handle_command_reply: tid=0 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 client.25774 shutdown 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 msgr2=0x5560b98fc9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98fc9a0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8a40028460 tx=0x7f8a400057c0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 shutdown_connections 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5560b996bd50 0x5560b996e170 crc :-1 s=READY pgs=46 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5560b98f5de0 0x5560b98fb6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5560b98eb780 0x5560b98fc9a0 secure :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8a40028460 tx=0x7f8a400057c0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 --2- 172.21.15.90:0/1547416280 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5560b98ec0a0 0x5560b98fb060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.148+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 >> 172.21.15.90:0/1547416280 conn(0x5560b984a6a0 msgr2=0x5560b98f2090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.149+0000 7f8a2f7fe700 0 client.25774 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.149+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 shutdown_connections 2024-04-06T14:09:56.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.149+0000 7f8a6272eb80 1 -- 172.21.15.90:0/1547416280 wait complete. 2024-04-06T14:09:56.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.149+0000 7f8a6272eb80 20 client.25774 trim_cache size 0 max 16384 2024-04-06T14:09:56.151 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 msgr2=0x7f8a3c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 0x7f8a3c0a37d0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f8a40005950 tx=0x7f8a4000b500 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 msgr2=0x7f8a5006d130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a5006d130 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f8a4800be20 tx=0x7f8a4800be50 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 shutdown_connections 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a3c0a1310 0x7f8a3c0a37d0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a50111720 0x7f8a5006d130 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a5010b720 0x7f8a5006c6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 --2- 172.21.15.90:0/25885550 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a5010c0e0 0x7f8a5006cbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 >> 172.21.15.90:0/25885550 conn(0x7f8a5006bd10 msgr2=0x7f8a5010e6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 shutdown_connections 2024-04-06T14:09:56.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.150+0000 7f8a5657a700 1 -- 172.21.15.90:0/25885550 wait complete. 2024-04-06T14:09:56.170 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:56.170 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:09:56.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 -- 172.21.15.90:0/2847659435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc0ffcc0 msgr2=0x7fc3bc103960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 --2- 172.21.15.90:0/2847659435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc103960 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fc3b4009f90 tx=0x7fc3b400b330 comp rx=0 tx=0).stop 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 -- 172.21.15.90:0/2847659435 shutdown_connections 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 --2- 172.21.15.90:0/2847659435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc103960 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 --2- 172.21.15.90:0/2847659435 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc105e60 0x7fc3bc106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 --2- 172.21.15.90:0/2847659435 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc0ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 -- 172.21.15.90:0/2847659435 >> 172.21.15.90:0/2847659435 conn(0x7fc3bc0761b0 msgr2=0x7fc3bc0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 -- 172.21.15.90:0/2847659435 shutdown_connections 2024-04-06T14:09:56.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.386+0000 7fc3c3034700 1 -- 172.21.15.90:0/2847659435 wait complete. 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 Processor -- start 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 -- start start 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc06a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 0x7fc3bc06ade0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3bc06b5b0 con 0x7fc3bc0ff220 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3bc06b720 con 0x7fc3bc0ffcc0 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c3034700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3bc1b2880 con 0x7fc3bc105e60 2024-04-06T14:09:56.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47538/0 (socket says 172.21.15.90:47538) 2024-04-06T14:09:56.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3bbfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc06a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.387+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/1182880059 learned_addr learned my addr 172.21.15.90:0/1182880059 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 0x7fc3bc06ade0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 -- 172.21.15.90:0/1182880059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc0ffcc0 msgr2=0x7fc3bc06a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc06a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 -- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 msgr2=0x7fc3bc072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.388+0000 7fc3c15d1700 1 -- 172.21.15.90:0/1182880059 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3b4009c40 con 0x7fc3bc105e60 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 0x7fc3bc06ade0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fc3b4009f90 tx=0x7fc3b4009500 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3b9ffb700 1 -- 172.21.15.90:0/1182880059 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b400e040 con 0x7fc3bc105e60 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3b9ffb700 1 -- 172.21.15.90:0/1182880059 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3b4005040 con 0x7fc3bc105e60 2024-04-06T14:09:56.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3bc19d850 con 0x7fc3bc105e60 2024-04-06T14:09:56.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3b9ffb700 1 -- 172.21.15.90:0/1182880059 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b401c660 con 0x7fc3bc105e60 2024-04-06T14:09:56.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.389+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3bc19dd50 con 0x7fc3bc105e60 2024-04-06T14:09:56.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.394+0000 7fc3cf1e8b80 1 Processor -- start 2024-04-06T14:09:56.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 -- start start 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd55120a0 0x55bfd551b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd551e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5469990 con 0x55bfd5511780 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5467350 con 0x55bfd55120a0 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.398+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5462c40 con 0x55bfd551bde0 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.399+0000 7fc3b9ffb700 1 -- 172.21.15.90:0/1182880059 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc3b40051b0 con 0x7fc3bc105e60 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.400+0000 7fc3b9ffb700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 0x7fc3a40a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.400+0000 7fc3bbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.400+0000 7fc3bbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47540/0 (socket says 172.21.15.90:47540) 2024-04-06T14:09:56.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.400+0000 7fc3bbfff700 1 -- 172.21.15.90:0/1956389448 learned_addr learned my addr 172.21.15.90:0/1956389448 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3b9ffb700 1 -- 172.21.15.90:0/1182880059 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3b40cd4c0 con 0x7fc3bc105e60 2024-04-06T14:09:56.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 -- 172.21.15.90:0/1956389448 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 msgr2=0x55bfd551e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd551e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 -- 172.21.15.90:0/1956389448 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd55120a0 msgr2=0x55bfd551b8a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd55120a0 0x55bfd551b8a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 -- 172.21.15.90:0/1956389448 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfd52e43b0 con 0x55bfd5511780 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.401+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac009d90 tx=0x7fc3ac011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f72d8149bc5f974 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.402+0000 7fc3a2ffd700 1 -- 172.21.15.90:0/1956389448 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ac01a440 con 0x55bfd5511780 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.402+0000 7fc3a2ffd700 1 -- 172.21.15.90:0/1956389448 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3ac01aa60 con 0x55bfd5511780 2024-04-06T14:09:56.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.402+0000 7fc3a2ffd700 1 -- 172.21.15.90:0/1956389448 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ac00f990 con 0x55bfd5511780 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1956389448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 msgr2=0x55bfd5511b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 secure :-1 s=READY pgs=549 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac009d90 tx=0x7fc3ac011720 comp rx=0 tx=0).stop 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1956389448 shutdown_connections 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd551e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5511b60 unknown :-1 s=CLOSED pgs=549 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1956389448 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd55120a0 0x55bfd551b8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.403+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1956389448 >> 172.21.15.90:0/1956389448 conn(0x55bfd54706a0 msgr2=0x55bfd550ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.404+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 0x7fc3a40a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.414+0000 7fc3c15d1700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 0x7fc3a40a3810 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fc3b40cac80 tx=0x7fc3b4009040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.415+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1956389448 shutdown_connections 2024-04-06T14:09:56.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.416+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1956389448 wait complete. 2024-04-06T14:09:56.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.417+0000 7fc3cf1e8b80 1 -- start start 2024-04-06T14:09:56.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.417+0000 7fc3cf1e8b80 1 Processor -- start 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd551b870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd5518bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5527b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd53fed80 con 0x55bfd5511780 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd53fea20 con 0x55bfd5519250 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd52e66a0 con 0x55bfd551bde0 2024-04-06T14:09:56.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.418+0000 7fc3cf1e8b80 10 client.0 _ll_register_callbacks cb 0x55bfd5408470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd5518bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd5518bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49298/0 (socket says 172.21.15.90:49298) 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/3519618300 learned_addr learned my addr 172.21.15.90:0/3519618300 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5527b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 -- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 msgr2=0x55bfd5518bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd5518bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 -- 172.21.15.90:0/3519618300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 msgr2=0x55bfd551b870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd551b870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 -- 172.21.15.90:0/3519618300 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3ac009a70 con 0x55bfd5519250 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5527b40 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac009d90 tx=0x7fc3ac01af90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4241242e7c90adf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.419+0000 7fc39bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ac01a640 con 0x55bfd5519250 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bfd5423380 con 0x55bfd5519250 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3ac01a7a0 con 0x55bfd5519250 2024-04-06T14:09:56.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3ac005280 con 0x55bfd5519250 2024-04-06T14:09:56.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc3cf1e8b80 10 client.25877 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.420+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bfd5290b60 con 0x55bfd5519250 2024-04-06T14:09:56.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3ac033020 con 0x55bfd5519250 2024-04-06T14:09:56.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3ac021030 con 0x55bfd5519250 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 10 client.25877 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 10 client.25877 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 20 client.25877 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd552bd70 0x55bfd552e190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 4 client.25877 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bfd545b700 con 0x55bfd552bd70 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.421+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd552bd70 0x55bfd552e190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.422+0000 7fc39bfff700 10 client.25877 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:56.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.422+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd552bd70 0x55bfd552e190 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=30c93c709419bf5c server_cookie=84ffc9aed626ebe8 in_seq=0 out_seq=0 2024-04-06T14:09:56.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.424+0000 7fc39bfff700 1 -- 172.21.15.90:0/3519618300 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55bfd545b700 con 0x55bfd552bd70 2024-04-06T14:09:56.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.424+0000 7fc39bfff700 10 client.25877 handle_command_reply: tid=0 2024-04-06T14:09:56.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.425+0000 7fc3cf1e8b80 1 client.25877 shutdown 2024-04-06T14:09:56.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 msgr2=0x55bfd5527b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5527b40 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fc3ac009d90 tx=0x7fc3ac01af90 comp rx=0 tx=0).stop 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 shutdown_connections 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd552bd70 0x55bfd552e190 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5527b40 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd551b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc39bfff700 0 client.25877 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3519618300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd5518bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 >> 172.21.15.90:0/3519618300 conn(0x55bfd54706a0 msgr2=0x55bfd550bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 shutdown_connections 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3519618300 wait complete. 2024-04-06T14:09:56.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.426+0000 7fc3cf1e8b80 20 client.25877 trim_cache size 0 max 16384 2024-04-06T14:09:56.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 Processor -- start 2024-04-06T14:09:56.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 -- start start 2024-04-06T14:09:56.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd5511780 0x55bfd550cc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5518b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd552e7f0 con 0x55bfd551bde0 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5517100 con 0x55bfd5519250 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd550d790 con 0x55bfd5511780 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3bbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5518b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3bbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47560/0 (socket says 172.21.15.90:47560) 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3bbfff700 1 -- 172.21.15.90:0/3189764765 learned_addr learned my addr 172.21.15.90:0/3189764765 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd5511780 0x55bfd550cc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/3189764765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd5511780 msgr2=0x55bfd550cc20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd5511780 0x55bfd550cc20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/3189764765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 msgr2=0x55bfd5518490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/3189764765 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfd5428ea0 con 0x55bfd5519250 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.446+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.447+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5518b70 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x55bfd552e770 tx=0x7fc3b003ebe0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=da0781afdd8bfd02 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.447+0000 7fc3a37fe700 1 -- 172.21.15.90:0/3189764765 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b0039040 con 0x55bfd5519250 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.447+0000 7fc3a37fe700 1 -- 172.21.15.90:0/3189764765 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3b0042070 con 0x55bfd5519250 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.447+0000 7fc3a37fe700 1 -- 172.21.15.90:0/3189764765 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b0032440 con 0x55bfd5519250 2024-04-06T14:09:56.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.448+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3189764765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 msgr2=0x55bfd5518b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.448+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5518b70 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x55bfd552e770 tx=0x7fc3b003ebe0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3189764765 shutdown_connections 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd551bde0 0x55bfd5518490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd5511780 0x55bfd550cc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/3189764765 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5518b70 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3189764765 >> 172.21.15.90:0/3189764765 conn(0x55bfd54706a0 msgr2=0x55bfd550c510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3189764765 shutdown_connections 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.449+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/3189764765 wait complete. 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 -- start start 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 Processor -- start 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5522a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd55215b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5521c90 con 0x55bfd5511780 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5521e00 con 0x55bfd5519250 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfd5521f40 con 0x55bfd551bde0 2024-04-06T14:09:56.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.450+0000 7fc3cf1e8b80 10 client.0 _ll_register_callbacks cb 0x55bfd5408470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52240/0 (socket says 172.21.15.90:52240) 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/1830125516 learned_addr learned my addr 172.21.15.90:0/1830125516 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 msgr2=0x55bfd55215b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd55215b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/1830125516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 msgr2=0x55bfd5522a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5522a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 -- 172.21.15.90:0/1830125516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3b0009d50 con 0x55bfd5519250 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc3c0dd0700 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fc3b0009cc0 tx=0x7fc3b0004800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c04cc8c560f22daf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.451+0000 7fc39bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3b0039050 con 0x55bfd5519250 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3b0004d20 con 0x55bfd5519250 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc3ac01aa60 con 0x55bfd5519250 2024-04-06T14:09:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3b0005020 con 0x55bfd5519250 2024-04-06T14:09:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc3cf1e8b80 10 client.25883 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bfd52e8cd0 con 0x55bfd5519250 2024-04-06T14:09:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.452+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3ac01aa60 con 0x55bfd5519250 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.453+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3b0005180 con 0x55bfd5519250 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.453+0000 7fc3cf1e8b80 10 client.25883 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.453+0000 7fc3cf1e8b80 10 client.25883 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.453+0000 7fc3cf1e8b80 20 client.25883 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd5591fb0 0x55bfd55943d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc3cf1e8b80 4 client.25883 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55bfd52b1cf0 con 0x55bfd5591fb0 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd5591fb0 0x55bfd55943d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc39bfff700 10 client.25883 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:56.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc3bbfff700 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd5591fb0 0x55bfd55943d0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=45b7220f5d5ed371 server_cookie=3c6eae0c9b0fb80c in_seq=0 out_seq=0 2024-04-06T14:09:56.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc39bfff700 1 -- 172.21.15.90:0/1830125516 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55bfd52b1cf0 con 0x55bfd5591fb0 2024-04-06T14:09:56.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.454+0000 7fc39bfff700 10 client.25883 handle_command_reply: tid=0 2024-04-06T14:09:56.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.455+0000 7fc3cf1e8b80 1 client.25883 shutdown 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.455+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 msgr2=0x55bfd5521070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.455+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fc3b0009cc0 tx=0x7fc3b0004800 comp rx=0 tx=0).stop 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 shutdown_connections 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bfd5591fb0 0x55bfd55943d0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfd551bde0 0x55bfd55215b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfd5511780 0x55bfd5522a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 --2- 172.21.15.90:0/1830125516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfd5519250 0x55bfd5521070 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 >> 172.21.15.90:0/1830125516 conn(0x55bfd54706a0 msgr2=0x55bfd551a400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc39bfff700 0 client.25883 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 shutdown_connections 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 1 -- 172.21.15.90:0/1830125516 wait complete. 2024-04-06T14:09:56.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.456+0000 7fc3cf1e8b80 20 client.25883 trim_cache size 0 max 16384 2024-04-06T14:09:56.458 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 msgr2=0x7fc3a40a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 0x7fc3a40a3810 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7fc3b40cac80 tx=0x7fc3b4009040 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 msgr2=0x7fc3bc06ade0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 0x7fc3bc06ade0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fc3b4009f90 tx=0x7fc3b4009500 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 shutdown_connections 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3a40a1350 0x7fc3a40a3810 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3bc105e60 0x7fc3bc06ade0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3bc0ff220 0x7fc3bc072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 --2- 172.21.15.90:0/1182880059 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3bc0ffcc0 0x7fc3bc06a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.457+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 >> 172.21.15.90:0/1182880059 conn(0x7fc3bc0761b0 msgr2=0x7fc3bc101470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.458+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 shutdown_connections 2024-04-06T14:09:56.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.458+0000 7fc3c3034700 1 -- 172.21.15.90:0/1182880059 wait complete. 2024-04-06T14:09:56.476 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:09:56.476 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:09:56.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.662+0000 7f61dead6700 1 -- 172.21.15.90:0/869798805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 msgr2=0x7f61d81150f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.662+0000 7f61dead6700 1 --2- 172.21.15.90:0/869798805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81150f0 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f61d000f430 tx=0x7f61d000f740 comp rx=0 tx=0).stop 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 -- 172.21.15.90:0/869798805 shutdown_connections 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 --2- 172.21.15.90:0/869798805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81150f0 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 --2- 172.21.15.90:0/869798805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d8109b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 --2- 172.21.15.90:0/869798805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d810a070 0x7f61d81127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 -- 172.21.15.90:0/869798805 >> 172.21.15.90:0/869798805 conn(0x7f61d806c250 msgr2=0x7f61d806c660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 -- 172.21.15.90:0/869798805 shutdown_connections 2024-04-06T14:09:56.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.663+0000 7f61dead6700 1 -- 172.21.15.90:0/869798805 wait complete. 2024-04-06T14:09:56.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.664+0000 7f61dead6700 1 Processor -- start 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 -- start start 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d810a070 0x7f61d81a6500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81aa8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d81aaee0 con 0x7f61d810a070 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d81ab050 con 0x7f61d8109750 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dead6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d81ab210 con 0x7f61d8112d00 2024-04-06T14:09:56.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52246/0 (socket says 172.21.15.90:52246) 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 -- 172.21.15.90:0/16412420 learned_addr learned my addr 172.21.15.90:0/16412420 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dd073700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81aa8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 -- 172.21.15.90:0/16412420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 msgr2=0x7f61d81aa8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81aa8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 -- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d810a070 msgr2=0x7f61d81a6500 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d810a070 0x7f61d81a6500 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.666+0000 7f61dc872700 1 -- 172.21.15.90:0/16412420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61d000f0b0 con 0x7f61d8109750 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61dc872700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009f00 tx=0x7f61cc00dbb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61d5ffb700 1 -- 172.21.15.90:0/16412420 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61cc004a30 con 0x7f61d8109750 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61d81ab490 con 0x7f61d8109750 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61d5ffb700 1 -- 172.21.15.90:0/16412420 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f61cc034070 con 0x7f61d8109750 2024-04-06T14:09:56.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61d5ffb700 1 -- 172.21.15.90:0/16412420 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61cc00c870 con 0x7f61d8109750 2024-04-06T14:09:56.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.667+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61d819d6e0 con 0x7f61d8109750 2024-04-06T14:09:56.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.671+0000 7f61eac8ab80 1 Processor -- start 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 -- start start 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a717780 0x55ff1a717b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a7180a0 0x55ff1a7218a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a66f990 con 0x55ff1a7180a0 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a66d350 con 0x55ff1a721de0 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a668c40 con 0x55ff1a717780 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61d7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61d7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52254/0 (socket says 172.21.15.90:52254) 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61d7fff700 1 -- 172.21.15.90:0/3257765160 learned_addr learned my addr 172.21.15.90:0/3257765160 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.672+0000 7f61d5ffb700 1 -- 172.21.15.90:0/16412420 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f61cc036070 con 0x7f61d8109750 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.673+0000 7f61dd073700 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a717780 0x55ff1a717b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.673+0000 7f61dc872700 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a7180a0 0x55ff1a7218a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.673+0000 7f61d5ffb700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 0x7f61c80a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 -- 172.21.15.90:0/3257765160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a717780 msgr2=0x55ff1a717b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a717780 0x55ff1a717b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 -- 172.21.15.90:0/3257765160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a7180a0 msgr2=0x55ff1a7218a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a7180a0 0x55ff1a7218a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 -- 172.21.15.90:0/3257765160 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff1a4ea3b0 con 0x55ff1a721de0 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d5ffb700 1 -- 172.21.15.90:0/16412420 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f61cc0eddd0 con 0x7f61d8109750 2024-04-06T14:09:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61d7fff700 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f61c40097a0 tx=0x7f61c4009ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c991e257eb035ea server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61c2ffd700 1 -- 172.21.15.90:0/3257765160 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61c4011830 con 0x55ff1a721de0 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61c2ffd700 1 -- 172.21.15.90:0/3257765160 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f61c4011e50 con 0x55ff1a721de0 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.674+0000 7f61c2ffd700 1 -- 172.21.15.90:0/3257765160 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61c400f2e0 con 0x55ff1a721de0 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 -- 172.21.15.90:0/3257765160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 msgr2=0x55ff1a7241d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f61c40097a0 tx=0x7f61c4009ab0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 -- 172.21.15.90:0/3257765160 shutdown_connections 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61dd073700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 0x7f61c80a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a7241d0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a717780 0x55ff1a717b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/3257765160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a7180a0 0x55ff1a7218a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61eac8ab80 1 -- 172.21.15.90:0/3257765160 >> 172.21.15.90:0/3257765160 conn(0x55ff1a6766a0 msgr2=0x55ff1a712a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.675+0000 7f61dd073700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 0x7f61c80a37d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f61d000dfd0 tx=0x7f61d000dd30 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.676+0000 7f61eac8ab80 1 -- 172.21.15.90:0/3257765160 shutdown_connections 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.676+0000 7f61eac8ab80 1 -- 172.21.15.90:0/3257765160 wait complete. 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.676+0000 7f61eac8ab80 1 -- start start 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 Processor -- start 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a721830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a721de0 0x55ff1a71f260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a604d80 con 0x55ff1a717780 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a604a20 con 0x55ff1a7180a0 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a4ec6a0 con 0x55ff1a721de0 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.677+0000 7f61eac8ab80 10 client.0 _ll_register_callbacks cb 0x55ff1a60e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52266/0 (socket says 172.21.15.90:52266) 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 -- 172.21.15.90:0/2509711641 learned_addr learned my addr 172.21.15.90:0/2509711641 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61dd073700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a721de0 0x55ff1a71f260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 -- 172.21.15.90:0/2509711641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a721de0 msgr2=0x55ff1a71f260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a721de0 0x55ff1a71f260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 -- 172.21.15.90:0/2509711641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 msgr2=0x55ff1a721830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a721830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 -- 172.21.15.90:0/2509711641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61c40093d0 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61d7fff700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f61c4010630 tx=0x7f61c4010710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2686aae00c5eaf91 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61b7fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61c4010e30 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f61c4011630 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ff1a629380 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f61c40117f0 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61eac8ab80 10 client.25895 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.678+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ff1a496b60 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f61c402f020 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f61c401ab60 con 0x55ff1a7180a0 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 10 client.25895 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 10 client.25895 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 20 client.25895 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a731820 0x55ff1a733c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 4 client.25895 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ff1a661700 con 0x55ff1a731820 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.679+0000 7f61dc872700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a731820 0x55ff1a733c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.680+0000 7f61dc872700 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a731820 0x55ff1a733c40 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5b4c921d12e8b947 server_cookie=8c5cfe65a911d55c in_seq=0 out_seq=0 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.680+0000 7f61b7fff700 10 client.25895 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.680+0000 7f61b7fff700 1 -- 172.21.15.90:0/2509711641 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ff1a661700 con 0x55ff1a731820 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.680+0000 7f61b7fff700 10 client.25895 handle_command_reply: tid=0 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.680+0000 7f61eac8ab80 1 client.25895 shutdown 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.681+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 msgr2=0x55ff1a71eb20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.681+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f61c4010630 tx=0x7f61c4010710 comp rx=0 tx=0).stop 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 shutdown_connections 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a731820 0x55ff1a733c40 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a721de0 0x55ff1a71f260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a721830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2509711641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a7180a0 0x55ff1a71eb20 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61b7fff700 0 client.25895 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:56.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 >> 172.21.15.90:0/2509711641 conn(0x55ff1a6766a0 msgr2=0x55ff1a711c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 shutdown_connections 2024-04-06T14:09:56.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.684+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2509711641 wait complete. 2024-04-06T14:09:56.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.685+0000 7f61eac8ab80 20 client.25895 trim_cache size 0 max 16384 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.702+0000 7f61eac8ab80 1 Processor -- start 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.702+0000 7f61eac8ab80 1 -- start start 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a717780 0x55ff1a713dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a71e3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a71eee0 con 0x55ff1a721de0 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a719650 con 0x55ff1a717780 2024-04-06T14:09:56.703 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a71e8f0 con 0x55ff1a7180a0 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47614/0 (socket says 172.21.15.90:47614) 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 -- 172.21.15.90:0/2615614658 learned_addr learned my addr 172.21.15.90:0/2615614658 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61d7fff700 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a71e3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 -- 172.21.15.90:0/2615614658 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 msgr2=0x55ff1a71e3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a71e3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 -- 172.21.15.90:0/2615614658 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a717780 msgr2=0x55ff1a713dc0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:09:56.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a717780 0x55ff1a713dc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 -- 172.21.15.90:0/2615614658 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff1a62eea0 con 0x55ff1a721de0 2024-04-06T14:09:56.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.703+0000 7f61dd073700 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f61d0018000 tx=0x7f61d001f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=816a7305e1a36061 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.704+0000 7f61c37fe700 1 -- 172.21.15.90:0/2615614658 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61d0012040 con 0x55ff1a721de0 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.704+0000 7f61c37fe700 1 -- 172.21.15.90:0/2615614658 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f61d000b710 con 0x55ff1a721de0 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.704+0000 7f61c37fe700 1 -- 172.21.15.90:0/2615614658 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61d0005920 con 0x55ff1a721de0 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2615614658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 msgr2=0x55ff1a728a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 secure :-1 s=READY pgs=550 cs=0 l=1 rev1=1 crypto rx=0x7f61d0018000 tx=0x7f61d001f000 comp rx=0 tx=0).stop 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2615614658 shutdown_connections 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a721de0 0x55ff1a728a60 unknown :-1 s=CLOSED pgs=550 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a717780 0x55ff1a713dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/2615614658 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a71e3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2615614658 >> 172.21.15.90:0/2615614658 conn(0x55ff1a6766a0 msgr2=0x55ff1a7385e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2615614658 shutdown_connections 2024-04-06T14:09:56.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- 172.21.15.90:0/2615614658 wait complete. 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.705+0000 7f61eac8ab80 1 -- start start 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 Processor -- start 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a727170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a7276b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a785730 con 0x55ff1a717780 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a7858a0 con 0x55ff1a721de0 2024-04-06T14:09:56.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff1a785a60 con 0x55ff1a7180a0 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61eac8ab80 10 client.0 _ll_register_callbacks cb 0x55ff1a60e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61dd073700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61dd073700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52284/0 (socket says 172.21.15.90:52284) 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.706+0000 7f61dd073700 1 -- 172.21.15.90:0/181426077 learned_addr learned my addr 172.21.15.90:0/181426077 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 -- 172.21.15.90:0/181426077 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 msgr2=0x55ff1a7276b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61d7fff700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a7276b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a7276b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 -- 172.21.15.90:0/181426077 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 msgr2=0x55ff1a727170 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a727170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 -- 172.21.15.90:0/181426077 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61d8115b50 con 0x55ff1a721de0 2024-04-06T14:09:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61d7fff700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a7276b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:09:56.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61dd073700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f61d000f080 tx=0x7f61d00050b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=78e2cd6a00ae62e2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:56.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61b7fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f61d003a070 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.707+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f61c4011e50 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f61d0027040 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f61d0012070 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 10 client.25898 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ff1a4eecd0 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f61c4011e50 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f61d003d020 con 0x55ff1a721de0 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 10 client.25898 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 10 client.25898 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 20 client.25898 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:56.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a786db0 0x55ff1a7891d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:56.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 4 client.25898 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:09:56.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.708+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ff1a4b7cf0 con 0x55ff1a786db0 2024-04-06T14:09:56.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.709+0000 7f61dc872700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a786db0 0x55ff1a7891d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:56.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.710+0000 7f61b7fff700 10 client.25898 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:56.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.710+0000 7f61dc872700 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a786db0 0x55ff1a7891d0 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e0025c4d75edcd2b server_cookie=1ce05c4e28173977 in_seq=0 out_seq=0 2024-04-06T14:09:56.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61b7fff700 1 -- 172.21.15.90:0/181426077 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ff1a4b7cf0 con 0x55ff1a786db0 2024-04-06T14:09:56.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61b7fff700 10 client.25898 handle_command_reply: tid=0 2024-04-06T14:09:56.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 client.25898 shutdown 2024-04-06T14:09:56.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 msgr2=0x55ff1a727bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f61d000f080 tx=0x7f61d00050b0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 shutdown_connections 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff1a721de0 0x55ff1a727bf0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff1a717780 0x55ff1a727170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ff1a786db0 0x55ff1a7891d0 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 --2- 172.21.15.90:0/181426077 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff1a7180a0 0x55ff1a7276b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 >> 172.21.15.90:0/181426077 conn(0x55ff1a6766a0 msgr2=0x55ff1a716770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.712+0000 7f61b7fff700 0 client.25898 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:56.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.713+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 shutdown_connections 2024-04-06T14:09:56.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.713+0000 7f61eac8ab80 1 -- 172.21.15.90:0/181426077 wait complete. 2024-04-06T14:09:56.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.713+0000 7f61eac8ab80 20 client.25898 trim_cache size 0 max 16384 2024-04-06T14:09:56.714 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 msgr2=0x7f61c80a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 0x7f61c80a37d0 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f61d000dfd0 tx=0x7f61d000dd30 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 msgr2=0x7f61d81a5fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009f00 tx=0x7f61cc00dbb0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 shutdown_connections 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f61c80a1310 0x7f61c80a37d0 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f61d8112d00 0x7f61d81aa8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f61d8109750 0x7f61d81a5fc0 secure :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f61cc009f00 tx=0x7f61cc00dbb0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 --2- 172.21.15.90:0/16412420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f61d810a070 0x7f61d81a6500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:56.715 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.714+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 >> 172.21.15.90:0/16412420 conn(0x7f61d806c250 msgr2=0x7f61d8070e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:56.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.717+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 shutdown_connections 2024-04-06T14:09:56.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:56.717+0000 7f61dead6700 1 -- 172.21.15.90:0/16412420 wait complete. 2024-04-06T14:09:56.732 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:56 smithi090 ceph-mon[43420]: pgmap v139: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 124 KiB/s rd, 43 MiB/s wr, 2.22k op/s 2024-04-06T14:09:56.732 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:56 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2556573788' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:57.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:56 smithi105 ceph-mon[46969]: pgmap v139: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 124 KiB/s rd, 43 MiB/s wr, 2.22k op/s 2024-04-06T14:09:57.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:56 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2556573788' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:57.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:56 smithi134 ceph-mon[47549]: pgmap v139: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 124 KiB/s rd, 43 MiB/s wr, 2.22k op/s 2024-04-06T14:09:57.270 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:56 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2556573788' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:09:57.735 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:09:57.735 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:09:57.736 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:09:57.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.911+0000 7f1bc8a34700 1 -- 172.21.15.90:0/3487655385 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc4108940 msgr2=0x7f1bc410bf70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.911+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/3487655385 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc4108940 0x7f1bc410bf70 secure :-1 s=READY pgs=551 cs=0 l=1 rev1=1 crypto rx=0x7f1bb80098f0 tx=0x7f1bb8009c00 comp rx=0 tx=0).stop 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 -- 172.21.15.90:0/3487655385 shutdown_connections 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/3487655385 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc410c6a0 0x7f1bc407a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/3487655385 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4107f90 0x7f1bc4108370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/3487655385 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc4108940 0x7f1bc410bf70 unknown :-1 s=CLOSED pgs=551 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 -- 172.21.15.90:0/3487655385 >> 172.21.15.90:0/3487655385 conn(0x7f1bc406be00 msgr2=0x7f1bc406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 -- 172.21.15.90:0/3487655385 shutdown_connections 2024-04-06T14:09:57.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.912+0000 7f1bc8a34700 1 -- 172.21.15.90:0/3487655385 wait complete. 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 Processor -- start 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 -- start start 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 0x7f1bc4075ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc410c6a0 0x7f1bc4076530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc4076d00 con 0x7f1bc410c6a0 2024-04-06T14:09:57.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc4076e70 con 0x7f1bc4107f90 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc8a34700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bc407a8d0 con 0x7f1bc4108940 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.913+0000 7f1bc259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52298/0 (socket says 172.21.15.90:52298) 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 -- 172.21.15.90:0/2684327831 learned_addr learned my addr 172.21.15.90:0/2684327831 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc410c6a0 0x7f1bc4076530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 0x7f1bc4075ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 -- 172.21.15.90:0/2684327831 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 msgr2=0x7f1bc4075ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 0x7f1bc4075ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 -- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc410c6a0 msgr2=0x7f1bc4076530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc410c6a0 0x7f1bc4076530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 -- 172.21.15.90:0/2684327831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1bb8009520 con 0x7f1bc4107f90 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 0x7f1bc4075ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc259c700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f1bbc00ef10 tx=0x7f1bbc00c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bb37fe700 1 -- 172.21.15.90:0/2684327831 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bbc00ca10 con 0x7f1bc4107f90 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1bc407ab20 con 0x7f1bc4107f90 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bb37fe700 1 -- 172.21.15.90:0/2684327831 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bbc00f040 con 0x7f1bc4107f90 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.914+0000 7f1bb37fe700 1 -- 172.21.15.90:0/2684327831 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bbc0146e0 con 0x7f1bc4107f90 2024-04-06T14:09:57.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.915+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bc407b050 con 0x7f1bc4107f90 2024-04-06T14:09:57.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.917+0000 7f1bb37fe700 1 -- 172.21.15.90:0/2684327831 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1bbc007500 con 0x7f1bc4107f90 2024-04-06T14:09:57.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.917+0000 7f1bb37fe700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 0x7f1bac0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.918+0000 7f1bb37fe700 1 -- 172.21.15.90:0/2684327831 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bbc0cd040 con 0x7f1bc4107f90 2024-04-06T14:09:57.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.918+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 0x7f1bac0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.919+0000 7f1bd4be8b80 1 Processor -- start 2024-04-06T14:09:57.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 -- start start 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46cc90be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46cc9a920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cbe8a10 con 0x55d46cc91120 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cbe63d0 con 0x55d46cc9ae60 2024-04-06T14:09:57.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cbe1cc0 con 0x55d46cc90800 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.920+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 0x7f1bac0a3750 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1bb8009fd0 tx=0x7f1bb8019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52306/0 (socket says 172.21.15.90:52306) 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/2713352904 learned_addr learned my addr 172.21.15.90:0/2713352904 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc259c700 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46cc90be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46cc9a920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/2713352904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 msgr2=0x55d46cc90be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46cc90be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/2713352904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 msgr2=0x55d46cc9a920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46cc9a920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/2713352904 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d46ca63430 con 0x55d46cc9ae60 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.921+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4009c70 tx=0x7f1bb4009f80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=38b10bd1fc3a9e27 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.922+0000 7f1bb0ff9700 1 -- 172.21.15.90:0/2713352904 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb4011d10 con 0x55d46cc9ae60 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.922+0000 7f1bb0ff9700 1 -- 172.21.15.90:0/2713352904 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb4011e70 con 0x55d46cc9ae60 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.922+0000 7f1bb0ff9700 1 -- 172.21.15.90:0/2713352904 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb4005280 con 0x55d46cc9ae60 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.922+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/2713352904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 msgr2=0x55d46cc9d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.922+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4009c70 tx=0x7f1bb4009f80 comp rx=0 tx=0).stop 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/2713352904 shutdown_connections 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc9d250 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46cc90be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/2713352904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46cc9a920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/2713352904 >> 172.21.15.90:0/2713352904 conn(0x55d46cbef720 msgr2=0x55d46cc8bb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/2713352904 shutdown_connections 2024-04-06T14:09:57.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/2713352904 wait complete. 2024-04-06T14:09:57.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 -- start start 2024-04-06T14:09:57.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.923+0000 7f1bd4be8b80 1 Processor -- start 2024-04-06T14:09:57.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc9a800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc98260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cb7de00 con 0x55d46cc90800 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cb7daa0 con 0x55d46cc9ae60 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46ca65720 con 0x55d46cc91120 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bd4be8b80 10 client.0 _ll_register_callbacks cb 0x55d46cb874f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bc1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bc1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49392/0 (socket says 172.21.15.90:49392) 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.924+0000 7f1bc1d9b700 1 -- 172.21.15.90:0/1598884085 learned_addr learned my addr 172.21.15.90:0/1598884085 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc98260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 -- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 msgr2=0x55d46cc98260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc98260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 -- 172.21.15.90:0/1598884085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 msgr2=0x55d46cc9a800 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc9a800 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 -- 172.21.15.90:0/1598884085 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bb4009920 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bc1d9b700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f1bb80098f0 tx=0x7f1bb8009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=40f6f5f0273e309c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1ba9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb8028070 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bb800bbc0 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d46cba2400 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1bb801c040 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bd4be8b80 10 client.25810 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d46ca0fbe0 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.925+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d46cba2400 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1bb804f4a0 con 0x55d46cc91120 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 10 client.25810 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:57.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 10 client.25810 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 20 client.25810 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cca1a10 0x55d46cca3eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 4 client.25810 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.926+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d46cbda780 con 0x55d46cca1a10 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.927+0000 7f1bc259c700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cca1a10 0x55d46cca3eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.927+0000 7f1bc259c700 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cca1a10 0x55d46cca3eb0 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fb74560d37b02d0 server_cookie=aa7e07efc68095cd in_seq=0 out_seq=0 2024-04-06T14:09:57.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.927+0000 7f1ba9ffb700 10 client.25810 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.928+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/1598884085 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d46cbda780 con 0x55d46cca1a10 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.928+0000 7f1ba9ffb700 10 client.25810 handle_command_reply: tid=0 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.928+0000 7f1bd4be8b80 1 client.25810 shutdown 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.928+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 msgr2=0x55d46cc97c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.928+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f1bb80098f0 tx=0x7f1bb8009260 comp rx=0 tx=0).stop 2024-04-06T14:09:57.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 shutdown_connections 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc98260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc9a800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cca1a10 0x55d46cca3eb0 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1598884085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc97c30 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 >> 172.21.15.90:0/1598884085 conn(0x55d46cbef720 msgr2=0x55d46cc8aca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1ba9ffb700 0 client.25810 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 shutdown_connections 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1598884085 wait complete. 2024-04-06T14:09:57.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.929+0000 7f1bd4be8b80 20 client.25810 trim_cache size 0 max 16384 2024-04-06T14:09:57.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 Processor -- start 2024-04-06T14:09:57.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 -- start start 2024-04-06T14:09:57.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc981c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc8b5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cc85da0 con 0x55d46cc90800 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cc9e550 con 0x55d46cc9ae60 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cc8c020 con 0x55d46cc91120 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bc259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc981c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc8b5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.952+0000 7f1bc2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc981c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47674/0 (socket says 172.21.15.90:47674) 2024-04-06T14:09:57.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52314/0 (socket says 172.21.15.90:52314) 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc259c700 1 -- 172.21.15.90:0/1380580029 learned_addr learned my addr 172.21.15.90:0/1380580029 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/1380580029 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 msgr2=0x55d46cc8b5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc8b5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/1380580029 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 msgr2=0x55d46cc981c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc981c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/1380580029 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d46cba7f20 con 0x55d46cc9ae60 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4011720 tx=0x7f1bb4004cf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=594d7e3377fc3796 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bb17fa700 1 -- 172.21.15.90:0/1380580029 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb4011b10 con 0x55d46cc9ae60 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bb17fa700 1 -- 172.21.15.90:0/1380580029 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb4011c70 con 0x55d46cc9ae60 2024-04-06T14:09:57.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.953+0000 7f1bb17fa700 1 -- 172.21.15.90:0/1380580029 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb40051c0 con 0x55d46cc9ae60 2024-04-06T14:09:57.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1380580029 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 msgr2=0x55d46cc8bae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4011720 tx=0x7f1bb4004cf0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1380580029 shutdown_connections 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cc8bae0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc90800 0x55d46cc981c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/1380580029 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc91120 0x55d46cc8b5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.954+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1380580029 >> 172.21.15.90:0/1380580029 conn(0x55d46cbef720 msgr2=0x55d46cc96540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1380580029 shutdown_connections 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/1380580029 wait complete. 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- start start 2024-04-06T14:09:57.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 Processor -- start 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46ccf79c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46ccf7f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cca7b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cca8120 con 0x55d46cc91120 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cca8260 con 0x55d46cc9ae60 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d46cca8420 con 0x55d46cc90800 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bd4be8b80 10 client.0 _ll_register_callbacks cb 0x55d46cb874f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:57.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.955+0000 7f1bc1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46ccf7f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cca7b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46ccf7f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47676/0 (socket says 172.21.15.90:47676) 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc1d9b700 1 -- 172.21.15.90:0/203885081 learned_addr learned my addr 172.21.15.90:0/203885081 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc259c700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46ccf79c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/203885081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 msgr2=0x55d46ccf79c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46ccf79c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/203885081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 msgr2=0x55d46ccf7f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46ccf7f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 -- 172.21.15.90:0/203885081 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bb401d040 con 0x55d46cc9ae60 2024-04-06T14:09:57.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bc2d9d700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cca7b50 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4005920 tx=0x7f1bb4003af0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=24d2e4ac0ffac6ca server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1ba9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bb4011b10 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1bb4011e70 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1bb4004e00 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1bb4004fc0 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bd4be8b80 10 client.25916 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.956+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d46ca67d50 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bb4011e70 con 0x55d46cc9ae60 2024-04-06T14:09:57.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1bb4030780 con 0x55d46cc9ae60 2024-04-06T14:09:57.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 10 client.25916 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 10 client.25916 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 20 client.25916 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cd005f0 0x55d46cd029e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 4 client.25916 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55d46ca30d70 con 0x55d46cd005f0 2024-04-06T14:09:57.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1bc259c700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cd005f0 0x55d46cd029e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:57.968 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.957+0000 7f1ba9ffb700 10 client.25916 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:57.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.961+0000 7f1bc259c700 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cd005f0 0x55d46cd029e0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=652a05f3bf3302e6 server_cookie=321003c7aa90e4d1 in_seq=0 out_seq=0 2024-04-06T14:09:57.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1ba9ffb700 1 -- 172.21.15.90:0/203885081 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55d46ca30d70 con 0x55d46cd005f0 2024-04-06T14:09:57.969 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1ba9ffb700 10 client.25916 handle_command_reply: tid=0 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 client.25916 shutdown 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 msgr2=0x55d46cca7b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cca7b50 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f1bb4005920 tx=0x7f1bb4003af0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 shutdown_connections 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d46cc9ae60 0x55d46cca7b50 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d46cd005f0 0x55d46cd029e0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d46cc90800 0x55d46ccf79c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 --2- 172.21.15.90:0/203885081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d46cc91120 0x55d46ccf7f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 >> 172.21.15.90:0/203885081 conn(0x55d46cbef720 msgr2=0x55d46cc963d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1ba9ffb700 0 client.25916 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 shutdown_connections 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 1 -- 172.21.15.90:0/203885081 wait complete. 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.962+0000 7f1bd4be8b80 20 client.25916 trim_cache size 0 max 16384 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:09:57.971 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "051b986c-3591-43e6-9ef2-75466acc7c55", 2024-04-06T14:09:57.972 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:09:57.972 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:57.972 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 msgr2=0x7f1bac0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 0x7f1bac0a3750 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1bb8009fd0 tx=0x7f1bb8019040 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 msgr2=0x7f1bc4079f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f1bbc00ef10 tx=0x7f1bbc00c3a0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 shutdown_connections 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1bac0a1290 0x7f1bac0a3750 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1bc410c6a0 0x7f1bc4076530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1bc4107f90 0x7f1bc4079f30 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 --2- 172.21.15.90:0/2684327831 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1bc4108940 0x7f1bc4075ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 >> 172.21.15.90:0/2684327831 conn(0x7f1bc406be00 msgr2=0x7f1bc4070c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.963+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 shutdown_connections 2024-04-06T14:09:57.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:57.964+0000 7f1bc8a34700 1 -- 172.21.15.90:0/2684327831 wait complete. 2024-04-06T14:09:57.982 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:09:57.982 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:09:58.028 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 250.0 s, 2866.9 tps, lat 34.824 ms stddev 41.878 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 -- 172.21.15.90:0/905214432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c1060b0 msgr2=0x7ff95c10dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 --2- 172.21.15.90:0/905214432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c1060b0 0x7ff95c10dc30 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7ff9540098f0 tx=0x7ff954009c00 comp rx=0 tx=0).stop 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 -- 172.21.15.90:0/905214432 shutdown_connections 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 --2- 172.21.15.90:0/905214432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c1060b0 0x7ff95c10dc30 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 --2- 172.21.15.90:0/905214432 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff95c100f90 0x7ff95c101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 --2- 172.21.15.90:0/905214432 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c101940 0x7ff95c105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.203+0000 7ff962a5b700 1 -- 172.21.15.90:0/905214432 >> 172.21.15.90:0/905214432 conn(0x7ff95c0fc7f0 msgr2=0x7ff95c0fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.220+0000 7ff962a5b700 1 -- 172.21.15.90:0/905214432 shutdown_connections 2024-04-06T14:09:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.222+0000 7ff962a5b700 1 -- 172.21.15.90:0/905214432 wait complete. 2024-04-06T14:09:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.223+0000 7ff962a5b700 1 Processor -- start 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 -- start start 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c100f90 0x7ff95c19de40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff95c10aba0 0x7ff95c10b050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff95c10b780 con 0x7ff95c10aba0 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff95c10b8f0 con 0x7ff95c100f90 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff962a5b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff95c1998f0 con 0x7ff95c1060b0 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49428/0 (socket says 172.21.15.90:49428) 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.224+0000 7ff9537fe700 1 -- 172.21.15.90:0/2224241912 learned_addr learned my addr 172.21.15.90:0/2224241912 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 -- 172.21.15.90:0/2224241912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c100f90 msgr2=0x7ff95c19de40 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c100f90 0x7ff95c19de40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 -- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff95c10aba0 msgr2=0x7ff95c10b050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff95c10aba0 0x7ff95c10b050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 -- 172.21.15.90:0/2224241912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff94c009d00 con 0x7ff95c1060b0 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff9537fe700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7ff94400e970 tx=0x7ff94400ed30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff959ffb700 1 -- 172.21.15.90:0/2224241912 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff94400cbd0 con 0x7ff95c1060b0 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.225+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff954009520 con 0x7ff95c1060b0 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.226+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff95c199e60 con 0x7ff95c1060b0 2024-04-06T14:09:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.226+0000 7ff959ffb700 1 -- 172.21.15.90:0/2224241912 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff94400cd30 con 0x7ff95c1060b0 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.226+0000 7ff959ffb700 1 -- 172.21.15.90:0/2224241912 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff944005580 con 0x7ff95c1060b0 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 Processor -- start 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 -- start start 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d738b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d742de0 0x556d0d7451d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d690990 con 0x556d0d738780 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d68e350 con 0x556d0d742de0 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.230+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d689c40 con 0x556d0d7390a0 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49444/0 (socket says 172.21.15.90:49444) 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 -- 172.21.15.90:0/1630840699 learned_addr learned my addr 172.21.15.90:0/1630840699 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 -- 172.21.15.90:0/1630840699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d742de0 msgr2=0x556d0d7451d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d742de0 0x556d0d7451d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 -- 172.21.15.90:0/1630840699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 msgr2=0x556d0d738b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d738b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 -- 172.21.15.90:0/1630840699 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556d0d50b3b0 con 0x556d0d7390a0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff94c00c5d0 tx=0x7ff94c00c8e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=20077f83186b8905 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff959ffb700 1 -- 172.21.15.90:0/2224241912 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff944004750 con 0x7ff95c1060b0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.231+0000 7ff9527fc700 1 -- 172.21.15.90:0/1630840699 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff94c00e050 con 0x556d0d7390a0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.232+0000 7ff9527fc700 1 -- 172.21.15.90:0/1630840699 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff94c00f040 con 0x556d0d7390a0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.232+0000 7ff9527fc700 1 -- 172.21.15.90:0/1630840699 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff94c007e80 con 0x556d0d7390a0 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.232+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1630840699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 msgr2=0x556d0d7428a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.232+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff94c00c5d0 tx=0x7ff94c00c8e0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.233+0000 7ff959ffb700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 0x7ff9480c57e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.233+0000 7ff959ffb700 1 -- 172.21.15.90:0/2224241912 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff944014070 con 0x7ff95c1060b0 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1630840699 shutdown_connections 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d742de0 0x556d0d7451d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d738b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1630840699 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d7390a0 0x556d0d7428a0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1630840699 >> 172.21.15.90:0/1630840699 conn(0x556d0d6976a0 msgr2=0x556d0d733a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff960ff8700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 0x7ff9480c57e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.234+0000 7ff960ff8700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 0x7ff9480c57e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7ff954009230 tx=0x7ff954005e00 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.235+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1630840699 shutdown_connections 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.235+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1630840699 wait complete. 2024-04-06T14:09:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.236+0000 7ff96ec0fb80 1 -- start start 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.236+0000 7ff96ec0fb80 1 Processor -- start 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d7427e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d7401f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d625d80 con 0x556d0d738780 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d625a20 con 0x556d0d7390a0 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d50d6a0 con 0x556d0d742de0 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff96ec0fb80 10 client.0 _ll_register_callbacks cb 0x556d0d62f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:52346/0 (socket says 172.21.15.90:52346) 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff9537fe700 1 -- 172.21.15.90:0/1084499984 learned_addr learned my addr 172.21.15.90:0/1084499984 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff960ff8700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d7401f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d7427e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff9537fe700 1 -- 172.21.15.90:0/1084499984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 msgr2=0x556d0d7401f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.237+0000 7ff9537fe700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d7401f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.238+0000 7ff9537fe700 1 -- 172.21.15.90:0/1084499984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 msgr2=0x556d0d7427e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.238+0000 7ff9537fe700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d7427e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.238+0000 7ff9537fe700 1 -- 172.21.15.90:0/1084499984 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff94c00c2b0 con 0x556d0d7390a0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.239+0000 7ff9537fe700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff9440996e0 tx=0x7ff94409a210 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e701ab672134ffdd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.239+0000 7ff93b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.239+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff944097d80 con 0x556d0d7390a0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.239+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff944099c40 con 0x556d0d7390a0 2024-04-06T14:09:58.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.240+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x556d0d64a380 con 0x556d0d7390a0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.240+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff944099e00 con 0x556d0d7390a0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.240+0000 7ff96ec0fb80 10 client.25922 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.240+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x556d0d4b7b60 con 0x556d0d7390a0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.240+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff9440b6070 con 0x556d0d7390a0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff9440a19d0 con 0x556d0d7390a0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 10 client.25922 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 10 client.25922 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 20 client.25922 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d754eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 4 client.25922 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x556d0d682700 con 0x556d0d752a90 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d754eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff93b7fe700 10 client.25922 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:58.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.241+0000 7ff95bfff700 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d754eb0 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=86248d5efaeacf9a server_cookie=f4e2b3334b3ef4b7 in_seq=0 out_seq=0 2024-04-06T14:09:58.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff93b7fe700 1 -- 172.21.15.90:0/1084499984 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x556d0d682700 con 0x556d0d752a90 2024-04-06T14:09:58.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff93b7fe700 10 client.25922 handle_command_reply: tid=0 2024-04-06T14:09:58.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 client.25922 shutdown 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 msgr2=0x556d0d73fb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7ff9440996e0 tx=0x7ff94409a210 comp rx=0 tx=0).stop 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 shutdown_connections 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d754eb0 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d7401f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff93b7fe700 0 client.25922 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d7427e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/1084499984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d73fb70 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.242+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 >> 172.21.15.90:0/1084499984 conn(0x556d0d6976a0 msgr2=0x556d0d732c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.243+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 shutdown_connections 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.243+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/1084499984 wait complete. 2024-04-06T14:09:58.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.243+0000 7ff96ec0fb80 20 client.25922 trim_cache size 0 max 16384 2024-04-06T14:09:58.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.264+0000 7ff96ec0fb80 1 Processor -- start 2024-04-06T14:09:58.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.264+0000 7ff96ec0fb80 1 -- start start 2024-04-06T14:09:58.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 0x556d0d749af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d73e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d73eee0 con 0x556d0d7390a0 2024-04-06T14:09:58.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d7407f0 con 0x556d0d738780 2024-04-06T14:09:58.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d740b00 con 0x556d0d742de0 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47724/0 (socket says 172.21.15.90:47724) 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.265+0000 7ff9537fe700 1 -- 172.21.15.90:0/161540292 learned_addr learned my addr 172.21.15.90:0/161540292 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff960ff8700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d73e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 -- 172.21.15.90:0/161540292 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 msgr2=0x556d0d73e9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d73e9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 -- 172.21.15.90:0/161540292 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 msgr2=0x556d0d749af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff95bfff700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 0x556d0d749af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 0x556d0d749af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 -- 172.21.15.90:0/161540292 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556d0d64fea0 con 0x556d0d7390a0 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff95bfff700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 0x556d0d749af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.266+0000 7ff9537fe700 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7ff944095ee0 tx=0x7ff944005910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ba680e2a2156a42f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff952ffd700 1 -- 172.21.15.90:0/161540292 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff944097d80 con 0x556d0d7390a0 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff952ffd700 1 -- 172.21.15.90:0/161540292 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff9440b6030 con 0x556d0d7390a0 2024-04-06T14:09:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff952ffd700 1 -- 172.21.15.90:0/161540292 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff9440a19d0 con 0x556d0d7390a0 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/161540292 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 msgr2=0x556d0d73e460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 secure :-1 s=READY pgs=552 cs=0 l=1 rev1=1 crypto rx=0x7ff944095ee0 tx=0x7ff944005910 comp rx=0 tx=0).stop 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/161540292 shutdown_connections 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d73e9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d738780 0x556d0d749af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/161540292 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d7390a0 0x556d0d73e460 unknown :-1 s=CLOSED pgs=552 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.267+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/161540292 >> 172.21.15.90:0/161540292 conn(0x556d0d6976a0 msgr2=0x556d0d73ae80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.268+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/161540292 shutdown_connections 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.268+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/161540292 wait complete. 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.268+0000 7ff96ec0fb80 1 -- start start 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.268+0000 7ff96ec0fb80 1 Processor -- start 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d749870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d747cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d748910 con 0x556d0d738780 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d748a80 con 0x556d0d7390a0 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556d0d7a68d0 con 0x556d0d742de0 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff96ec0fb80 10 client.0 _ll_register_callbacks cb 0x556d0d62f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff960ff8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d747cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff960ff8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49466/0 (socket says 172.21.15.90:49466) 2024-04-06T14:09:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff960ff8700 1 -- 172.21.15.90:0/3973935287 learned_addr learned my addr 172.21.15.90:0/3973935287 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff95bfff700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d749870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 -- 172.21.15.90:0/3973935287 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 msgr2=0x556d0d748230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 -- 172.21.15.90:0/3973935287 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 msgr2=0x556d0d749870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d749870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 -- 172.21.15.90:0/3973935287 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556d0d4b7b60 con 0x556d0d7390a0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff960ff8700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff9537fe700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d747cf0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff9440996e0 tx=0x7ff944099a90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a8e1aadccbacf3a8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff93b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.269+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff944098ed0 con 0x556d0d7390a0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff944098ae0 con 0x556d0d7390a0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff94c00f040 con 0x556d0d7390a0 2024-04-06T14:09:58.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff944099030 con 0x556d0d7390a0 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff96ec0fb80 10 client.25925 fetch_fsmap learned FSMap version 24 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x556d0d50fcd0 con 0x556d0d7390a0 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.270+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff94c00f040 con 0x556d0d7390a0 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff9440a7030 con 0x556d0d7390a0 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 10 client.25925 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 10 client.25925 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 20 client.25925 populate_metadata read hostname 'smithi090' 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d7a99a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 4 client.25925 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:09:58.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.271+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x556d0d4d8cf0 con 0x556d0d752a90 2024-04-06T14:09:58.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff95bfff700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d7a99a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:09:58.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff93b7fe700 10 client.25925 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff95bfff700 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d7a99a0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=41a9133c7fb8cef0 server_cookie=db5ca48250f9c6b9 in_seq=0 out_seq=0 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff93b7fe700 1 -- 172.21.15.90:0/3973935287 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x556d0d4d8cf0 con 0x556d0d752a90 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff93b7fe700 10 client.25925 handle_command_reply: tid=0 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.272+0000 7ff96ec0fb80 1 client.25925 shutdown 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 msgr2=0x556d0d747cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d747cf0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff9440996e0 tx=0x7ff944099a90 comp rx=0 tx=0).stop 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 shutdown_connections 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556d0d752a90 0x556d0d7a99a0 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556d0d742de0 0x556d0d748230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556d0d738780 0x556d0d749870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 --2- 172.21.15.90:0/3973935287 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556d0d7390a0 0x556d0d747cf0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 >> 172.21.15.90:0/3973935287 conn(0x556d0d6976a0 msgr2=0x556d0d73e3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff93b7fe700 0 client.25925 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:09:58.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 shutdown_connections 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 1 -- 172.21.15.90:0/3973935287 wait complete. 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.273+0000 7ff96ec0fb80 20 client.25925 trim_cache size 0 max 16384 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "051b986c-3591-43e6-9ef2-75466acc7c55": { 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "051b986c-3591-43e6-9ef2-75466acc7c55", 2024-04-06T14:09:58.277 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:09:58.278 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:09:58.278 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:09:58.278 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:09:58.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 msgr2=0x7ff9480c57e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 0x7ff9480c57e0 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7ff954009230 tx=0x7ff954005e00 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 msgr2=0x7ff95c19e3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7ff94400e970 tx=0x7ff94400ed30 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 shutdown_connections 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff9480c3320 0x7ff9480c57e0 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff95c10aba0 0x7ff95c10b050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff95c100f90 0x7ff95c19de40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 --2- 172.21.15.90:0/2224241912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff95c1060b0 0x7ff95c19e3a0 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 >> 172.21.15.90:0/2224241912 conn(0x7ff95c0fc7f0 msgr2=0x7ff95c10d960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 shutdown_connections 2024-04-06T14:09:58.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:09:58.274+0000 7ff962a5b700 1 -- 172.21.15.90:0/2224241912 wait complete. 2024-04-06T14:09:58.290 INFO:tasks.cephfs.filesystem:scrub status for tag:051b986c-3591-43e6-9ef2-75466acc7c55 - {'path': '/', 'tag': '051b986c-3591-43e6-9ef2-75466acc7c55', 'options': 'recursive,force'} 2024-04-06T14:09:58.636 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:58 smithi090 ceph-mon[43420]: pgmap v140: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:09:58.637 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:58 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:09:58.637 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:58 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:58.637 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:58 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:09:58.637 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:09:58 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:58 smithi134 ceph-mon[47549]: pgmap v140: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:09:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:58 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:09:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:58 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:58 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:09:58.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:09:58 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:58.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:58 smithi105 ceph-mon[46969]: pgmap v140: 97 pgs: 97 active+clean; 6.5 GiB data, 9.3 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:09:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:58 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:09:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:58 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:09:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:58 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:09:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:09:58 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:00.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:00 smithi105 ceph-mon[46969]: pgmap v141: 97 pgs: 97 active+clean; 6.5 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 889 op/s 2024-04-06T14:10:00.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:00 smithi105 ceph-mon[46969]: overall HEALTH_OK 2024-04-06T14:10:00.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:00 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:10:01.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:00 smithi090 ceph-mon[43420]: pgmap v141: 97 pgs: 97 active+clean; 6.5 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 889 op/s 2024-04-06T14:10:01.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:00 smithi090 ceph-mon[43420]: overall HEALTH_OK 2024-04-06T14:10:01.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:00 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:10:01.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:00 smithi134 ceph-mon[47549]: pgmap v141: 97 pgs: 97 active+clean; 6.5 GiB data, 9.4 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 889 op/s 2024-04-06T14:10:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:00 smithi134 ceph-mon[47549]: overall HEALTH_OK 2024-04-06T14:10:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:00 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:10:02.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:02 smithi105 ceph-mon[46969]: pgmap v142: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 19 MiB/s wr, 1.05k op/s 2024-04-06T14:10:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 255.0 s, 1448.2 tps, lat 66.048 ms stddev 132.687 2024-04-06T14:10:03.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:02 smithi090 ceph-mon[43420]: pgmap v142: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 19 MiB/s wr, 1.05k op/s 2024-04-06T14:10:03.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:02 smithi134 ceph-mon[47549]: pgmap v142: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 19 MiB/s wr, 1.05k op/s 2024-04-06T14:10:04.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:04 smithi105 ceph-mon[46969]: pgmap v143: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 14 MiB/s wr, 829 op/s 2024-04-06T14:10:05.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:04 smithi090 ceph-mon[43420]: pgmap v143: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 14 MiB/s wr, 829 op/s 2024-04-06T14:10:05.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:04 smithi134 ceph-mon[47549]: pgmap v143: 97 pgs: 97 active+clean; 6.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 14 MiB/s wr, 829 op/s 2024-04-06T14:10:06.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:06 smithi105 ceph-mon[46969]: pgmap v144: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 34 MiB/s wr, 2.24k op/s 2024-04-06T14:10:07.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:06 smithi090 ceph-mon[43420]: pgmap v144: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 34 MiB/s wr, 2.24k op/s 2024-04-06T14:10:07.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:06 smithi134 ceph-mon[47549]: pgmap v144: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 34 MiB/s wr, 2.24k op/s 2024-04-06T14:10:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 260.0 s, 2754.0 tps, lat 38.156 ms stddev 63.007 2024-04-06T14:10:08.901 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:08 smithi134 ceph-mon[47549]: pgmap v145: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 26 MiB/s wr, 1.80k op/s 2024-04-06T14:10:08.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:08 smithi105 ceph-mon[46969]: pgmap v145: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 26 MiB/s wr, 1.80k op/s 2024-04-06T14:10:09.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:08 smithi090 ceph-mon[43420]: pgmap v145: 97 pgs: 97 active+clean; 6.6 GiB data, 9.6 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 26 MiB/s wr, 1.80k op/s 2024-04-06T14:10:10.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:10 smithi105 ceph-mon[46969]: pgmap v146: 97 pgs: 97 active+clean; 6.6 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 28 MiB/s wr, 1.89k op/s 2024-04-06T14:10:11.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:10 smithi090 ceph-mon[43420]: pgmap v146: 97 pgs: 97 active+clean; 6.6 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 28 MiB/s wr, 1.89k op/s 2024-04-06T14:10:11.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:10 smithi134 ceph-mon[47549]: pgmap v146: 97 pgs: 97 active+clean; 6.6 GiB data, 9.7 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 28 MiB/s wr, 1.89k op/s 2024-04-06T14:10:13.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 265.0 s, 2118.6 tps, lat 47.179 ms stddev 55.700 2024-04-06T14:10:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:12 smithi090 ceph-mon[43420]: pgmap v147: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 30 MiB/s wr, 1.96k op/s 2024-04-06T14:10:13.151 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:12 smithi134 ceph-mon[47549]: pgmap v147: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 30 MiB/s wr, 1.96k op/s 2024-04-06T14:10:13.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:12 smithi105 ceph-mon[46969]: pgmap v147: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 30 MiB/s wr, 1.96k op/s 2024-04-06T14:10:14.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:13 smithi090 ceph-mon[43420]: pgmap v148: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 26 MiB/s wr, 1.71k op/s 2024-04-06T14:10:14.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:13 smithi134 ceph-mon[47549]: pgmap v148: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 26 MiB/s wr, 1.71k op/s 2024-04-06T14:10:14.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:13 smithi105 ceph-mon[46969]: pgmap v148: 97 pgs: 97 active+clean; 6.6 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 26 MiB/s wr, 1.71k op/s 2024-04-06T14:10:16.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:16 smithi105 ceph-mon[46969]: pgmap v149: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 36 MiB/s wr, 2.15k op/s 2024-04-06T14:10:17.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:16 smithi090 ceph-mon[43420]: pgmap v149: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 36 MiB/s wr, 2.15k op/s 2024-04-06T14:10:17.147 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:16 smithi134 ceph-mon[47549]: pgmap v149: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 36 MiB/s wr, 2.15k op/s 2024-04-06T14:10:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 270.0 s, 2573.3 tps, lat 38.750 ms stddev 44.299 2024-04-06T14:10:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:18 smithi105 ceph-mon[46969]: pgmap v150: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 740 op/s 2024-04-06T14:10:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:18 smithi090 ceph-mon[43420]: pgmap v150: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 740 op/s 2024-04-06T14:10:19.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:18 smithi134 ceph-mon[47549]: pgmap v150: 97 pgs: 97 active+clean; 6.7 GiB data, 9.9 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 740 op/s 2024-04-06T14:10:20.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:20 smithi105 ceph-mon[46969]: pgmap v151: 97 pgs: 97 active+clean; 6.7 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 17 MiB/s wr, 832 op/s 2024-04-06T14:10:21.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:20 smithi090 ceph-mon[43420]: pgmap v151: 97 pgs: 97 active+clean; 6.7 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 17 MiB/s wr, 832 op/s 2024-04-06T14:10:21.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:20 smithi134 ceph-mon[47549]: pgmap v151: 97 pgs: 97 active+clean; 6.7 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 17 MiB/s wr, 832 op/s 2024-04-06T14:10:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 275.0 s, 2299.2 tps, lat 42.470 ms stddev 48.411 2024-04-06T14:10:23.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:22 smithi090 ceph-mon[43420]: pgmap v152: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 23 MiB/s wr, 960 op/s 2024-04-06T14:10:23.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:22 smithi134 ceph-mon[47549]: pgmap v152: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 23 MiB/s wr, 960 op/s 2024-04-06T14:10:23.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:22 smithi105 ceph-mon[46969]: pgmap v152: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 23 MiB/s wr, 960 op/s 2024-04-06T14:10:24.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:23 smithi134 ceph-mon[47549]: pgmap v153: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 19 MiB/s wr, 749 op/s 2024-04-06T14:10:24.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:23 smithi105 ceph-mon[46969]: pgmap v153: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 19 MiB/s wr, 749 op/s 2024-04-06T14:10:24.293 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:23 smithi090 ceph-mon[43420]: pgmap v153: 97 pgs: 97 active+clean; 6.8 GiB data, 10 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 19 MiB/s wr, 749 op/s 2024-04-06T14:10:26.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:26 smithi105 ceph-mon[46969]: pgmap v154: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 44 MiB/s wr, 2.46k op/s 2024-04-06T14:10:27.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:26 smithi090 ceph-mon[43420]: pgmap v154: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 44 MiB/s wr, 2.46k op/s 2024-04-06T14:10:27.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:26 smithi134 ceph-mon[47549]: pgmap v154: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 44 MiB/s wr, 2.46k op/s 2024-04-06T14:10:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 280.0 s, 2152.4 tps, lat 47.589 ms stddev 101.064 2024-04-06T14:10:28.291 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:10:28.292 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:10:28.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.517+0000 7fc3ffb88700 1 -- 172.21.15.90:0/335887342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 msgr2=0x7fc3f80ff9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.517+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/335887342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f80ff9c0 secure :-1 s=READY pgs=553 cs=0 l=1 rev1=1 crypto rx=0x7fc3f000c210 tx=0x7fc3f000c520 comp rx=0 tx=0).stop 2024-04-06T14:10:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.528+0000 7fc3ffb88700 1 -- 172.21.15.90:0/335887342 shutdown_connections 2024-04-06T14:10:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.528+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/335887342 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f8104700 0x7fc3f810c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.528+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/335887342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f80ff9c0 unknown :-1 s=CLOSED pgs=553 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.528+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/335887342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f80fff90 0x7fc3f8103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.528+0000 7fc3ffb88700 1 -- 172.21.15.90:0/335887342 >> 172.21.15.90:0/335887342 conn(0x7fc3f80761b0 msgr2=0x7fc3f80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.540+0000 7fc3ffb88700 1 -- 172.21.15.90:0/335887342 shutdown_connections 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.542+0000 7fc3ffb88700 1 -- 172.21.15.90:0/335887342 wait complete. 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 Processor -- start 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 -- start start 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f806b3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f806be50 0x7fc3f8071750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3f8071d50 con 0x7fc3f80ff5e0 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.543+0000 7fc3ffb88700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3f8071ec0 con 0x7fc3f806be50 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3ffb88700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3f8072080 con 0x7fc3f8104700 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52310/0 (socket says 172.21.15.90:52310) 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 -- 172.21.15.90:0/758238665 learned_addr learned my addr 172.21.15.90:0/758238665 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fe125700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f806be50 0x7fc3f8071750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd924700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f806b3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 -- 172.21.15.90:0/758238665 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f806be50 msgr2=0x7fc3f8071750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f806be50 0x7fc3f8071750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 -- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 msgr2=0x7fc3f806b3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f806b3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 -- 172.21.15.90:0/758238665 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3f000ae10 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3fd123700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fc3e800b550 tx=0x7fc3e800b860 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.544+0000 7fc3eeffd700 1 -- 172.21.15.90:0/758238665 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e8011830 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.545+0000 7fc3eeffd700 1 -- 172.21.15.90:0/758238665 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3e8011e50 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.545+0000 7fc3eeffd700 1 -- 172.21.15.90:0/758238665 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e8010de0 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.545+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3f8072360 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.545+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3f8072820 con 0x7fc3f8104700 2024-04-06T14:10:28.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.547+0000 7fc3eeffd700 1 -- 172.21.15.90:0/758238665 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc3e8010480 con 0x7fc3f8104700 2024-04-06T14:10:28.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.548+0000 7fc3eeffd700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 0x7fc3e40a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.548+0000 7fc3eeffd700 1 -- 172.21.15.90:0/758238665 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3e80cc600 con 0x7fc3f8104700 2024-04-06T14:10:28.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.552+0000 7fc40bd3cb80 1 Processor -- start 2024-04-06T14:10:28.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.553+0000 7fc3fd924700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 0x7fc3e40a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 -- start start 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b58d8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b58dde0 0x55cf7b5901d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b4db990 con 0x55cf7b5840a0 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b4d9350 con 0x55cf7b58dde0 2024-04-06T14:10:28.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.554+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b4d4c40 con 0x55cf7b583780 2024-04-06T14:10:28.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52314/0 (socket says 172.21.15.90:52314) 2024-04-06T14:10:28.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 -- 172.21.15.90:0/1461058776 learned_addr learned my addr 172.21.15.90:0/1461058776 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fe125700 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b58dde0 0x55cf7b5901d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 -- 172.21.15.90:0/1461058776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b58dde0 msgr2=0x55cf7b5901d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b58dde0 0x55cf7b5901d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 -- 172.21.15.90:0/1461058776 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 msgr2=0x55cf7b58d8a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:10:28.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b58d8a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.555+0000 7fc3fd924700 1 -- 172.21.15.90:0/1461058776 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf7b3563b0 con 0x55cf7b583780 2024-04-06T14:10:28.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.557+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fc3f000c210 tx=0x7fc3f0003d20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e745f07fa44c82b8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc3e3fff700 1 -- 172.21.15.90:0/1461058776 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f0005480 con 0x55cf7b583780 2024-04-06T14:10:28.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc3e3fff700 1 -- 172.21.15.90:0/1461058776 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3f00074a0 con 0x55cf7b583780 2024-04-06T14:10:28.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc3e3fff700 1 -- 172.21.15.90:0/1461058776 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f0005aa0 con 0x55cf7b583780 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1461058776 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 msgr2=0x55cf7b583b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc3fd924700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 0x7fc3e40a3810 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fc3f0029000 tx=0x7fc3f00209a0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fc3f000c210 tx=0x7fc3f0003d20 comp rx=0 tx=0).stop 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1461058776 shutdown_connections 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b58dde0 0x55cf7b5901d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b583b60 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1461058776 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b58d8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.558+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1461058776 >> 172.21.15.90:0/1461058776 conn(0x55cf7b4e26a0 msgr2=0x55cf7b57ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.559+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1461058776 shutdown_connections 2024-04-06T14:10:28.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.559+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1461058776 wait complete. 2024-04-06T14:10:28.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.559+0000 7fc40bd3cb80 1 -- start start 2024-04-06T14:10:28.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.559+0000 7fc40bd3cb80 1 Processor -- start 2024-04-06T14:10:28.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b58ac90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b5840a0 0x55cf7b58b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b470d80 con 0x55cf7b58dde0 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b470a20 con 0x55cf7b5840a0 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b3586a0 con 0x55cf7b583780 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc40bd3cb80 10 client.0 _ll_register_callbacks cb 0x55cf7b47a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc3fe125700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc3fe125700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47240/0 (socket says 172.21.15.90:47240) 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc3fe125700 1 -- 172.21.15.90:0/1447726617 learned_addr learned my addr 172.21.15.90:0/1447726617 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.560+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b58ac90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 -- 172.21.15.90:0/1447726617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 msgr2=0x55cf7b58ac90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b58ac90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 -- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b5840a0 msgr2=0x55cf7b58b1d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b5840a0 0x55cf7b58b1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 -- 172.21.15.90:0/1447726617 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3f810caf0 con 0x55cf7b58dde0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3fe125700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4010470 tx=0x7fc3f4010830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=51baafbea7e6a2ff server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3e17fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f40094e0 con 0x55cf7b58dde0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cf7b495380 con 0x55cf7b58dde0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.561+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3f4009e20 con 0x55cf7b58dde0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3f4014800 con 0x55cf7b58dde0 2024-04-06T14:10:28.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 10 client.16212 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cf7b302b60 con 0x55cf7b58dde0 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55cf7b495380 con 0x55cf7b58dde0 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3f401dba0 con 0x55cf7b58dde0 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 10 client.16212 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 10 client.16212 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 20 client.16212 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b5aec00 0x55cf7b5b1020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 4 client.16212 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cf7b4cd700 con 0x55cf7b5aec00 2024-04-06T14:10:28.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.562+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b5aec00 0x55cf7b5b1020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.563+0000 7fc3e17fa700 10 client.16212 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.563+0000 7fc3fd924700 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b5aec00 0x55cf7b5b1020 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=45b58f6b8ee0c5ad server_cookie=82af42e975e92597 in_seq=0 out_seq=0 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.564+0000 7fc3e17fa700 1 -- 172.21.15.90:0/1447726617 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cf7b4cd700 con 0x55cf7b5aec00 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.564+0000 7fc3e17fa700 10 client.16212 handle_command_reply: tid=0 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.564+0000 7fc40bd3cb80 1 client.16212 shutdown 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 msgr2=0x55cf7b58b710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 secure :-1 s=READY pgs=554 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4010470 tx=0x7fc3f4010830 comp rx=0 tx=0).stop 2024-04-06T14:10:28.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 shutdown_connections 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b58b710 unknown :-1 s=CLOSED pgs=554 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b5aec00 0x55cf7b5b1020 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b583780 0x55cf7b58ac90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/1447726617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b5840a0 0x55cf7b58b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 >> 172.21.15.90:0/1447726617 conn(0x55cf7b4e26a0 msgr2=0x55cf7b57dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.565+0000 7fc3e17fa700 0 client.16212 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.566+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 shutdown_connections 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.566+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/1447726617 wait complete. 2024-04-06T14:10:28.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.566+0000 7fc40bd3cb80 20 client.16212 trim_cache size 0 max 16384 2024-04-06T14:10:28.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.589+0000 7fc40bd3cb80 1 Processor -- start 2024-04-06T14:10:28.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 -- start start 2024-04-06T14:10:28.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b57e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b57e8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b58d780 con 0x55cf7b5840a0 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b57f650 con 0x55cf7b583780 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b5763c0 con 0x55cf7b58dde0 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52330/0 (socket says 172.21.15.90:52330) 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 -- 172.21.15.90:0/2861644902 learned_addr learned my addr 172.21.15.90:0/2861644902 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fd924700 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b57e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 -- 172.21.15.90:0/2861644902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 msgr2=0x55cf7b57e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b57e3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 -- 172.21.15.90:0/2861644902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 msgr2=0x55cf7b57e8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b57e8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.591+0000 7fc3fe125700 1 -- 172.21.15.90:0/2861644902 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf7b49aea0 con 0x55cf7b58dde0 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.592+0000 7fc3fe125700 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4010cb0 tx=0x7fc3f4010ce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9f1c283766ba754b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.592+0000 7fc3ecda5700 1 -- 172.21.15.90:0/2861644902 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f4009ee0 con 0x55cf7b58dde0 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.592+0000 7fc3ecda5700 1 -- 172.21.15.90:0/2861644902 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3f401ccf0 con 0x55cf7b58dde0 2024-04-06T14:10:28.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.592+0000 7fc3ecda5700 1 -- 172.21.15.90:0/2861644902 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3f40039a0 con 0x55cf7b58dde0 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2861644902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 msgr2=0x55cf7b587f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7fc3f4010cb0 tx=0x7fc3f4010ce0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2861644902 shutdown_connections 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b58dde0 0x55cf7b587f30 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b57e3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2861644902 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b5840a0 0x55cf7b57e8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2861644902 >> 172.21.15.90:0/2861644902 conn(0x55cf7b4e26a0 msgr2=0x55cf7b589580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2861644902 shutdown_connections 2024-04-06T14:10:28.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2861644902 wait complete. 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.593+0000 7fc40bd3cb80 1 -- start start 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 Processor -- start 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b5a5b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b5a45d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b5a4d10 con 0x55cf7b58dde0 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b5a4e80 con 0x55cf7b583780 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf7b602a40 con 0x55cf7b5840a0 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc40bd3cb80 10 client.0 _ll_register_callbacks cb 0x55cf7b47a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fd123700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fd123700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52336/0 (socket says 172.21.15.90:52336) 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fd123700 1 -- 172.21.15.90:0/2499213686 learned_addr learned my addr 172.21.15.90:0/2499213686 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fe125700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b5a45d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fd123700 1 -- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 msgr2=0x55cf7b5a5b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.594+0000 7fc3fd123700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b5a5b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3fd123700 1 -- 172.21.15.90:0/2499213686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 msgr2=0x55cf7b5a45d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3fd123700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b5a45d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3fd123700 1 -- 172.21.15.90:0/2499213686 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55cf7b302b60 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3fd123700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc3e80994f0 tx=0x7fc3e8099330 comp rx=0 tx=0).ready entity=mon.1 client_cookie=12df8b39d8dd41c7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e8011b90 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fc3e80b9280 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc3f00074a0 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3e80997f0 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 10 client.25849 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cf7b35acd0 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3f00074a0 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3e8098ce0 con 0x55cf7b5840a0 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 10 client.25849 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:28.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 10 client.25849 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 20 client.25849 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b604200 0x55cf7b606620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 4 client.25849 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.595+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55cf7b323cf0 con 0x55cf7b604200 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc3fd924700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b604200 0x55cf7b606620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc3fd924700 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b604200 0x55cf7b606620 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b7cfbf8f711697b7 server_cookie=61b0f01f4806f269 in_seq=0 out_seq=0 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc3e17fa700 10 client.25849 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc3e17fa700 1 -- 172.21.15.90:0/2499213686 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55cf7b323cf0 con 0x55cf7b604200 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc3e17fa700 10 client.25849 handle_command_reply: tid=0 2024-04-06T14:10:28.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc40bd3cb80 1 client.25849 shutdown 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 msgr2=0x55cf7b5a4070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc3e80994f0 tx=0x7fc3e8099330 comp rx=0 tx=0).stop 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.596+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 shutdown_connections 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf7b58dde0 0x55cf7b5a45d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf7b583780 0x55cf7b5a5b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf7b604200 0x55cf7b606620 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 --2- 172.21.15.90:0/2499213686 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf7b5840a0 0x55cf7b5a4070 secure :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fc3e80994f0 tx=0x7fc3e8099330 comp rx=0 tx=0).stop 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 >> 172.21.15.90:0/2499213686 conn(0x55cf7b4e26a0 msgr2=0x55cf7b582770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc3e17fa700 0 client.25849 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 shutdown_connections 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.597+0000 7fc40bd3cb80 1 -- 172.21.15.90:0/2499213686 wait complete. 2024-04-06T14:10:28.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.599+0000 7fc40bd3cb80 20 client.25849 trim_cache size 0 max 16384 2024-04-06T14:10:28.600 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:10:28.600 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:10:28.600 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:10:28.600 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 msgr2=0x7fc3e40a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 0x7fc3e40a3810 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7fc3f0029000 tx=0x7fc3f00209a0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 msgr2=0x7fc3f806b910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fc3e800b550 tx=0x7fc3e800b860 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 shutdown_connections 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e40a1350 0x7fc3e40a3810 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3f806be50 0x7fc3f8071750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3f80ff5e0 0x7fc3f806b3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 --2- 172.21.15.90:0/758238665 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3f8104700 0x7fc3f806b910 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 >> 172.21.15.90:0/758238665 conn(0x7fc3f80761b0 msgr2=0x7fc3f8102940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 shutdown_connections 2024-04-06T14:10:28.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.600+0000 7fc3ffb88700 1 -- 172.21.15.90:0/758238665 wait complete. 2024-04-06T14:10:28.616 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:10:28.616 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/3723888081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd01004f0 msgr2=0x7f6fd01049b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/3723888081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd01004f0 0x7f6fd01049b0 secure :-1 s=READY pgs=555 cs=0 l=1 rev1=1 crypto rx=0x7f6fbc009990 tx=0x7f6fbc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/3723888081 shutdown_connections 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/3723888081 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd0104fd0 0x7f6fd0105480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/3723888081 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd00fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/3723888081 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd01004f0 0x7f6fd01049b0 unknown :-1 s=CLOSED pgs=555 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/3723888081 >> 172.21.15.90:0/3723888081 conn(0x7f6fd00fb5e0 msgr2=0x7f6fd00fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:28.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/3723888081 shutdown_connections 2024-04-06T14:10:28.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.794+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/3723888081 wait complete. 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 Processor -- start 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 -- start start 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd0113be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd01004f0 0x7f6fd0114160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 0x7f6fd010ce20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd010d5f0 con 0x7f6fd0104fd0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd010d760 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd6ef0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd010d920 con 0x7f6fd01004f0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd548d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 0x7f6fd010ce20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd4c8c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd0113be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd548d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 0x7f6fd010ce20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47282/0 (socket says 172.21.15.90:47282) 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.795+0000 7f6fd548d700 1 -- 172.21.15.90:0/2662509720 learned_addr learned my addr 172.21.15.90:0/2662509720 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 -- 172.21.15.90:0/2662509720 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd01004f0 msgr2=0x7f6fd0114160 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd01004f0 0x7f6fd0114160 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 -- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 msgr2=0x7f6fd010ce20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 0x7f6fd010ce20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 -- 172.21.15.90:0/2662509720 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fbc0095f0 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd4c8c700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd0113be0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f6fc400e9f0 tx=0x7f6fc400ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fc400eee0 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6fc400cbd0 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fd0111770 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.796+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fc4018680 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.797+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fd0111c80 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.798+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fb40052f0 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.799+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6fc4010430 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.799+0000 7f6fcdffb700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 0x7f6fc00a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:28.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.800+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f6fc4014070 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.800+0000 7f6fcffff700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 0x7f6fc00a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:28.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.800+0000 7f6fcffff700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 0x7f6fc00a3920 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f6fbc0098e0 tx=0x7f6fbc000bc0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:28.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:28.804+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6fc4052ab0 con 0x7f6fd00ffbd0 2024-04-06T14:10:28.899 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:28 smithi134 ceph-mon[47549]: pgmap v155: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 34 MiB/s wr, 2.02k op/s 2024-04-06T14:10:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:28 smithi105 ceph-mon[46969]: pgmap v155: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 34 MiB/s wr, 2.02k op/s 2024-04-06T14:10:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:28 smithi090 ceph-mon[43420]: pgmap v155: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 34 MiB/s wr, 2.02k op/s 2024-04-06T14:10:29.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.155+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6fb4005160 con 0x7f6fd00ffbd0 2024-04-06T14:10:29.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.156+0000 7f6fcdffb700 1 -- 172.21.15.90:0/2662509720 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f6fc4096170 con 0x7f6fd00ffbd0 2024-04-06T14:10:29.155 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:10:29.155 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 msgr2=0x7f6fc00a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 0x7f6fc00a3920 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f6fbc0098e0 tx=0x7f6fbc000bc0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 msgr2=0x7f6fd0113be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd0113be0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f6fc400e9f0 tx=0x7f6fc400ed00 comp rx=0 tx=0).stop 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 shutdown_connections 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6fc00a1460 0x7f6fc00a3920 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6fd0104fd0 0x7f6fd010ce20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fd00ffbd0 0x7f6fd0113be0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 --2- 172.21.15.90:0/2662509720 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6fd01004f0 0x7f6fd0114160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.159+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 >> 172.21.15.90:0/2662509720 conn(0x7f6fd00fb5e0 msgr2=0x7f6fd00fc600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.160+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 shutdown_connections 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.160+0000 7f6fd6ef0700 1 -- 172.21.15.90:0/2662509720 wait complete. 2024-04-06T14:10:29.160 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:10:29.175 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:10:29.175 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.353+0000 7fb867741700 1 -- 172.21.15.90:0/2346768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107d50 msgr2=0x7fb8601141f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.353+0000 7fb867741700 1 --2- 172.21.15.90:0/2346768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107d50 0x7fb8601141f0 secure :-1 s=READY pgs=556 cs=0 l=1 rev1=1 crypto rx=0x7fb8500098f0 tx=0x7fb850009c00 comp rx=0 tx=0).stop 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.353+0000 7fb867741700 1 -- 172.21.15.90:0/2346768231 shutdown_connections 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.353+0000 7fb867741700 1 --2- 172.21.15.90:0/2346768231 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107d50 0x7fb8601141f0 unknown :-1 s=CLOSED pgs=556 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.353+0000 7fb867741700 1 --2- 172.21.15.90:0/2346768231 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86010d830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.354+0000 7fb867741700 1 --2- 172.21.15.90:0/2346768231 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb860107390 0x7fb860107810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.354+0000 7fb867741700 1 -- 172.21.15.90:0/2346768231 >> 172.21.15.90:0/2346768231 conn(0x7fb860076ba0 msgr2=0x7fb860105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.354+0000 7fb867741700 1 -- 172.21.15.90:0/2346768231 shutdown_connections 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.354+0000 7fb867741700 1 -- 172.21.15.90:0/2346768231 wait complete. 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.354+0000 7fb867741700 1 Processor -- start 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 -- start start 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107390 0x7fb8600722e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb860107d50 0x7fb860072860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb86006bb80 con 0x7fb860107390 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb86006bcf0 con 0x7fb860107d50 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb867741700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb86006beb0 con 0x7fb86010d450 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52360/0 (socket says 172.21.15.90:52360) 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb8654dd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107390 0x7fb8600722e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 -- 172.21.15.90:0/75177396 learned_addr learned my addr 172.21.15.90:0/75177396 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 -- 172.21.15.90:0/75177396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb860107d50 msgr2=0x7fb860072860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb860107d50 0x7fb860072860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 -- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107390 msgr2=0x7fb8600722e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107390 0x7fb8600722e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.355+0000 7fb865cde700 1 -- 172.21.15.90:0/75177396 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb850009520 con 0x7fb86010d450 2024-04-06T14:10:29.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.356+0000 7fb865cde700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fb850009230 tx=0x7fb850005e40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.359+0000 7fb8567fc700 1 -- 172.21.15.90:0/75177396 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb85001c070 con 0x7fb86010d450 2024-04-06T14:10:29.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.359+0000 7fb8567fc700 1 -- 172.21.15.90:0/75177396 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb850005040 con 0x7fb86010d450 2024-04-06T14:10:29.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.359+0000 7fb8567fc700 1 -- 172.21.15.90:0/75177396 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb850004b90 con 0x7fb86010d450 2024-04-06T14:10:29.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.360+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb86006c130 con 0x7fb86010d450 2024-04-06T14:10:29.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.360+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb860070050 con 0x7fb86010d450 2024-04-06T14:10:29.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.364+0000 7fb8567fc700 1 -- 172.21.15.90:0/75177396 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb8500051b0 con 0x7fb86010d450 2024-04-06T14:10:29.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.366+0000 7fb8567fc700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 0x7fb84c0a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.366+0000 7fb8654dd700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 0x7fb84c0a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.366+0000 7fb8567fc700 1 -- 172.21.15.90:0/75177396 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb8500cd890 con 0x7fb86010d450 2024-04-06T14:10:29.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.366+0000 7fb8654dd700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 0x7fb84c0a3920 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fb85c006fd0 tx=0x7fb85c009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 Processor -- start 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 -- start start 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d2266fb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d2267c1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d225c7990 con 0x557d2266f780 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d225c5350 con 0x557d22679de0 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d225c0c40 con 0x557d226700a0 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb864cdc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb864cdc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52366/0 (socket says 172.21.15.90:52366) 2024-04-06T14:10:29.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.367+0000 7fb864cdc700 1 -- 172.21.15.90:0/2526944721 learned_addr learned my addr 172.21.15.90:0/2526944721 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.368+0000 7fb864cdc700 1 -- 172.21.15.90:0/2526944721 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 msgr2=0x557d2267c1d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:10:29.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.368+0000 7fb864cdc700 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d2267c1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.368+0000 7fb864cdc700 1 -- 172.21.15.90:0/2526944721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 msgr2=0x557d2266fb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.368+0000 7fb864cdc700 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d2266fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.368+0000 7fb864cdc700 1 -- 172.21.15.90:0/2526944721 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d224423b0 con 0x557d226700a0 2024-04-06T14:10:29.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.371+0000 7fb864cdc700 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fb85800e460 tx=0x7fb85800e770 comp rx=0 tx=0).ready entity=mon.1 client_cookie=68b44f1488980d12 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.374+0000 7fb84b7fe700 1 -- 172.21.15.90:0/2526944721 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb858018070 con 0x557d226700a0 2024-04-06T14:10:29.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.374+0000 7fb84b7fe700 1 -- 172.21.15.90:0/2526944721 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb85800f040 con 0x557d226700a0 2024-04-06T14:10:29.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.374+0000 7fb84b7fe700 1 -- 172.21.15.90:0/2526944721 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb858004280 con 0x557d226700a0 2024-04-06T14:10:29.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2526944721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 msgr2=0x557d226798a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fb85800e460 tx=0x7fb85800e770 comp rx=0 tx=0).stop 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2526944721 shutdown_connections 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d2267c1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d2266fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2526944721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d226798a0 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2526944721 >> 172.21.15.90:0/2526944721 conn(0x557d225ce6a0 msgr2=0x557d2266aa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2526944721 shutdown_connections 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.375+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2526944721 wait complete. 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 -- start start 2024-04-06T14:10:29.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 Processor -- start 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 0x557d22676d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2255cd80 con 0x557d22679de0 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2255ca20 con 0x557d2266f780 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d224446a0 con 0x557d226700a0 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb8738f5b80 10 client.0 _ll_register_callbacks cb 0x557d22566470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb865cde700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb865cde700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47320/0 (socket says 172.21.15.90:47320) 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.376+0000 7fb865cde700 1 -- 172.21.15.90:0/3894305649 learned_addr learned my addr 172.21.15.90:0/3894305649 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb8654dd700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 0x557d22676d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb864cdc700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 -- 172.21.15.90:0/3894305649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 msgr2=0x557d22677250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 -- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 msgr2=0x557d22676d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 0x557d22676d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 -- 172.21.15.90:0/3894305649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb85800e140 con 0x557d22679de0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb864cdc700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb8654dd700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 0x557d22676d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb865cde700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7fb85009a700 tx=0x7fb85009b180 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7909c340fbe8c83 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb848ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:10:29.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb850005690 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb8500a2df0 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.377+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557d22581380 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb8500b32b0 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 10 client.16230 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x557d223eeb60 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb8500b7080 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb850099ef0 con 0x557d22679de0 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 10 client.16230 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 10 client.16230 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 20 client.16230 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d22689750 0x557d2268bb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 4 client.16230 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.378+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557d225b9700 con 0x557d22689750 2024-04-06T14:10:29.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.379+0000 7fb8654dd700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d22689750 0x557d2268bb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.379+0000 7fb8654dd700 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d22689750 0x557d2268bb40 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=46fac81558e1e479 server_cookie=8ea6fe84b085bdab in_seq=0 out_seq=0 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.379+0000 7fb848ff9700 10 client.16230 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb848ff9700 1 -- 172.21.15.90:0/3894305649 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x557d225b9700 con 0x557d22689750 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb848ff9700 10 client.16230 handle_command_reply: tid=0 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 client.16230 shutdown 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 msgr2=0x557d22677790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 secure :-1 s=READY pgs=557 cs=0 l=1 rev1=1 crypto rx=0x7fb85009a700 tx=0x7fb85009b180 comp rx=0 tx=0).stop 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 shutdown_connections 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d22689750 0x557d2268bb40 crc :-1 s=READY pgs=47 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d22679de0 0x557d22677790 unknown :-1 s=CLOSED pgs=557 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d2266f780 0x557d22676d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb848ff9700 0 client.16230 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/3894305649 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 >> 172.21.15.90:0/3894305649 conn(0x557d225ce6a0 msgr2=0x557d22669c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 shutdown_connections 2024-04-06T14:10:29.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 1 -- 172.21.15.90:0/3894305649 wait complete. 2024-04-06T14:10:29.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.380+0000 7fb8738f5b80 20 client.16230 trim_cache size 0 max 16384 2024-04-06T14:10:29.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.396+0000 7fb8738f5b80 1 Processor -- start 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.396+0000 7fb8738f5b80 1 -- start start 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d226765d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d226711f0 con 0x557d2266f780 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d22676b10 con 0x557d22679de0 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d22679780 con 0x557d226700a0 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb864cdc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d226765d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb865cde700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb865cde700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48462/0 (socket says 172.21.15.90:48462) 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb864cdc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52372/0 (socket says 172.21.15.90:52372) 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb865cde700 1 -- 172.21.15.90:0/1131646739 learned_addr learned my addr 172.21.15.90:0/1131646739 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 -- 172.21.15.90:0/1131646739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 msgr2=0x557d22677420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 -- 172.21.15.90:0/1131646739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 msgr2=0x557d22673f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb8654dd700 1 -- 172.21.15.90:0/1131646739 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d22586ea0 con 0x557d2266f780 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.397+0000 7fb865cde700 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb8654dd700 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d226765d0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x557d2268c3b0 tx=0x7fb85c011000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=866490cc4f2c945d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb84bfff700 1 -- 172.21.15.90:0/1131646739 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb85c0093e0 con 0x557d2266f780 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb84bfff700 1 -- 172.21.15.90:0/1131646739 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb85c01b040 con 0x557d2266f780 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb84bfff700 1 -- 172.21.15.90:0/1131646739 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb85c015b90 con 0x557d2266f780 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb8738f5b80 1 -- 172.21.15.90:0/1131646739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 msgr2=0x557d226765d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.398+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d226765d0 secure :-1 s=READY pgs=558 cs=0 l=1 rev1=1 crypto rx=0x557d2268c3b0 tx=0x7fb85c011000 comp rx=0 tx=0).stop 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 -- 172.21.15.90:0/1131646739 shutdown_connections 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d22679de0 0x557d22673f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d226765d0 unknown :-1 s=CLOSED pgs=558 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/1131646739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d226700a0 0x557d22677420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 -- 172.21.15.90:0/1131646739 >> 172.21.15.90:0/1131646739 conn(0x557d225ce6a0 msgr2=0x557d22676d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 -- 172.21.15.90:0/1131646739 shutdown_connections 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 -- 172.21.15.90:0/1131646739 wait complete. 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 -- start start 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.399+0000 7fb8738f5b80 1 Processor -- start 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d22680370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d226700a0 0x557d226808b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2267f090 con 0x557d2266f780 2024-04-06T14:10:29.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2267f1d0 con 0x557d226700a0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d2267f390 con 0x557d22679de0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8738f5b80 10 client.0 _ll_register_callbacks cb 0x557d22566470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52378/0 (socket says 172.21.15.90:52378) 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 -- 172.21.15.90:0/2265653372 learned_addr learned my addr 172.21.15.90:0/2265653372 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb8654dd700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d22680370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb864cdc700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d226700a0 0x557d226808b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 -- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d226700a0 msgr2=0x557d226808b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d226700a0 0x557d226808b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 -- 172.21.15.90:0/2265653372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 msgr2=0x557d22680370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d22680370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.400+0000 7fb865cde700 1 -- 172.21.15.90:0/2265653372 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557d223eeb60 con 0x557d22679de0 2024-04-06T14:10:29.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb865cde700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fb850096f00 tx=0x7fb85009e970 comp rx=0 tx=0).ready entity=mon.1 client_cookie=56f299a746c280b4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb85009aa80 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb85800f040 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7fb8500a2df0 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb85000b8a0 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 10 client.25867 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fb85c00e9d0 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb85800f040 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb8500cb230 con 0x557d22679de0 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 10 client.25867 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 10 client.25867 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 20 client.25867 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d226de380 0x557d226e07a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 4 client.25867 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.401+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x557d2240fcf0 con 0x557d226de380 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8654dd700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d226de380 0x557d226e07a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8654dd700 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d226de380 0x557d226e07a0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=bab885106570b4fd server_cookie=34c898b868234846 in_seq=0 out_seq=0 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb848ff9700 10 client.25867 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb848ff9700 1 -- 172.21.15.90:0/2265653372 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x557d2240fcf0 con 0x557d226de380 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb848ff9700 10 client.25867 handle_command_reply: tid=0 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 client.25867 shutdown 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 msgr2=0x557d2267eac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fb850096f00 tx=0x7fb85009e970 comp rx=0 tx=0).stop 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 shutdown_connections 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x557d226de380 0x557d226e07a0 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557d22679de0 0x557d2267eac0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557d2266f780 0x557d22680370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 --2- 172.21.15.90:0/2265653372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557d226700a0 0x557d226808b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 >> 172.21.15.90:0/2265653372 conn(0x557d225ce6a0 msgr2=0x557d2266f000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb848ff9700 0 client.25867 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 shutdown_connections 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.402+0000 7fb8738f5b80 1 -- 172.21.15.90:0/2265653372 wait complete. 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb8738f5b80 20 client.25867 trim_cache size 0 max 16384 2024-04-06T14:10:29.408 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 msgr2=0x7fb84c0a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 0x7fb84c0a3920 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fb85c006fd0 tx=0x7fb85c009190 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 msgr2=0x7fb86006b3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fb850009230 tx=0x7fb850005e40 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 shutdown_connections 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb84c0a1460 0x7fb84c0a3920 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb86010d450 0x7fb86006b3b0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb860107390 0x7fb8600722e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 --2- 172.21.15.90:0/75177396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb860107d50 0x7fb860072860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 >> 172.21.15.90:0/75177396 conn(0x7fb860076ba0 msgr2=0x7fb860077b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 shutdown_connections 2024-04-06T14:10:29.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.403+0000 7fb867741700 1 -- 172.21.15.90:0/75177396 wait complete. 2024-04-06T14:10:29.417 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:10:29.417 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 -- 172.21.15.90:0/2898748635 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d4107390 msgr2=0x7f39d4107810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 --2- 172.21.15.90:0/2898748635 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d4107390 0x7f39d4107810 secure :-1 s=READY pgs=559 cs=0 l=1 rev1=1 crypto rx=0x7f39c8009990 tx=0x7f39c8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 -- 172.21.15.90:0/2898748635 shutdown_connections 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 --2- 172.21.15.90:0/2898748635 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107d50 0x7f39d41141f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 --2- 172.21.15.90:0/2898748635 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d410d450 0x7f39d410d830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 --2- 172.21.15.90:0/2898748635 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d4107390 0x7f39d4107810 unknown :-1 s=CLOSED pgs=559 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 -- 172.21.15.90:0/2898748635 >> 172.21.15.90:0/2898748635 conn(0x7f39d4076ba0 msgr2=0x7f39d4105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 -- 172.21.15.90:0/2898748635 shutdown_connections 2024-04-06T14:10:29.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.587+0000 7f39d89b8700 1 -- 172.21.15.90:0/2898748635 wait complete. 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 Processor -- start 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 -- start start 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 0x7f39d41a35a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d4107d50 0x7f39d41a3ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39d419dd30 con 0x7f39d410d450 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39d419dea0 con 0x7f39d4107d50 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d89b8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f39d419e060 con 0x7f39d4107390 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47356/0 (socket says 172.21.15.90:47356) 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3769981742 learned_addr learned my addr 172.21.15.90:0/3769981742 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d259c700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 0x7f39d41a35a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.588+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d4107d50 0x7f39d41a3ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.589+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3769981742 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 msgr2=0x7f39d41a35a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.589+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 0x7f39d41a35a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.589+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3769981742 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d4107d50 msgr2=0x7f39d41a3ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.589+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d4107d50 0x7f39d41a3ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.589+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3769981742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f39c80095f0 con 0x7f39d410d450 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.590+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f39cc00b780 tx=0x7f39cc00ba90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.590+0000 7f39d259c700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 0x7f39d41a35a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.590+0000 7f39c37fe700 1 -- 172.21.15.90:0/3769981742 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39cc010860 con 0x7f39d410d450 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.590+0000 7f39c37fe700 1 -- 172.21.15.90:0/3769981742 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f39cc010e80 con 0x7f39d410d450 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.590+0000 7f39c37fe700 1 -- 172.21.15.90:0/3769981742 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39cc007370 con 0x7f39d410d450 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.591+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f39d419e340 con 0x7f39d410d450 2024-04-06T14:10:29.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.591+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39d41aa590 con 0x7f39d410d450 2024-04-06T14:10:29.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.594+0000 7f39c37fe700 1 -- 172.21.15.90:0/3769981742 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f39cc00f5b0 con 0x7f39d410d450 2024-04-06T14:10:29.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.595+0000 7f39c37fe700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 0x7f39bc0a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.595+0000 7f39d259c700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 0x7f39bc0a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.595+0000 7f39c37fe700 1 -- 172.21.15.90:0/3769981742 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f39cc0cca10 con 0x7f39d410d450 2024-04-06T14:10:29.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.595+0000 7f39e4b6cb80 1 Processor -- start 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39d259c700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 0x7f39bc0a3920 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f39c4005950 tx=0x7f39c40058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 -- start start 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb60a0 0x5617d0fbf8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc21d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0f0d990 con 0x5617d0fb60a0 2024-04-06T14:10:29.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0f0b350 con 0x5617d0fb5780 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0f06c40 con 0x5617d0fbfde0 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.596+0000 7f39d259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d2d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc21d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48488/0 (socket says 172.21.15.90:48488) 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 -- 172.21.15.90:0/1618556852 learned_addr learned my addr 172.21.15.90:0/1618556852 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb60a0 0x5617d0fbf8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 -- 172.21.15.90:0/1618556852 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 msgr2=0x5617d0fc21d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc21d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 -- 172.21.15.90:0/1618556852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb60a0 msgr2=0x5617d0fbf8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb60a0 0x5617d0fbf8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d259c700 1 -- 172.21.15.90:0/1618556852 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5617d0d883b0 con 0x5617d0fb5780 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.597+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc21d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.598+0000 7f39d259c700 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f39c400ff00 tx=0x7f39c4012000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b78a5d715bd2e0c4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.598+0000 7f39c0ff9700 1 -- 172.21.15.90:0/1618556852 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c4017040 con 0x5617d0fb5780 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.598+0000 7f39c0ff9700 1 -- 172.21.15.90:0/1618556852 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f39c4003b40 con 0x5617d0fb5780 2024-04-06T14:10:29.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.598+0000 7f39c0ff9700 1 -- 172.21.15.90:0/1618556852 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c401fd10 con 0x5617d0fb5780 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/1618556852 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 msgr2=0x5617d0fb5b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f39c400ff00 tx=0x7f39c4012000 comp rx=0 tx=0).stop 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/1618556852 shutdown_connections 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc21d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb5780 0x5617d0fb5b60 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/1618556852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb60a0 0x5617d0fbf8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/1618556852 >> 172.21.15.90:0/1618556852 conn(0x5617d0f146a0 msgr2=0x5617d0fb0a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/1618556852 shutdown_connections 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/1618556852 wait complete. 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.599+0000 7f39e4b6cb80 1 -- start start 2024-04-06T14:10:29.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 Processor -- start 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 0x5617d0fbf860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 0x5617d0fbd290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0ea2d80 con 0x5617d0fbfde0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0ea2a20 con 0x5617d0fb60a0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0d8a6a0 con 0x5617d0fb5780 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39e4b6cb80 10 client.0 _ll_register_callbacks cb 0x5617d0eac470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39d2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 0x5617d0fbd290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39d1d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39d259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 0x5617d0fbf860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39d1d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48502/0 (socket says 172.21.15.90:48502) 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.600+0000 7f39d1d9b700 1 -- 172.21.15.90:0/194101369 learned_addr learned my addr 172.21.15.90:0/194101369 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 -- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 msgr2=0x5617d0fbf860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 0x5617d0fbf860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 -- 172.21.15.90:0/194101369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 msgr2=0x5617d0fbcba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 -- 172.21.15.90:0/194101369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39d41aa590 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d259c700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 0x5617d0fbf860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 0x5617d0fbd290 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f39cc0ca130 tx=0x7f39cc0994d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7be0334e8911c780 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39cc099af0 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f39cc09d920 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5617d0ec7380 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f39cc0a1f70 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 10 client.16248 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f39c400fbe0 con 0x5617d0fbfde0 2024-04-06T14:10:29.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5617d0ec7380 con 0x5617d0fbfde0 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f39cc0a1ae0 con 0x5617d0fbfde0 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 10 client.16248 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 10 client.16248 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.601+0000 7f39e4b6cb80 20 client.16248 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d0fe0880 0x5617d0fe2ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39e4b6cb80 4 client.16248 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5617d0eff700 con 0x5617d0fe0880 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39d259c700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d0fe0880 0x5617d0fe2ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39b9ffb700 10 client.16248 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:10:29.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.602+0000 7f39d259c700 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d0fe0880 0x5617d0fe2ca0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=6aa02b6dc8874bc4 server_cookie=b06eac3ba222fddd in_seq=0 out_seq=0 2024-04-06T14:10:29.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.603+0000 7f39b9ffb700 1 -- 172.21.15.90:0/194101369 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5617d0eff700 con 0x5617d0fe0880 2024-04-06T14:10:29.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.603+0000 7f39b9ffb700 10 client.16248 handle_command_reply: tid=0 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 client.16248 shutdown 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 msgr2=0x5617d0fbd290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 0x5617d0fbd290 secure :-1 s=READY pgs=561 cs=0 l=1 rev1=1 crypto rx=0x7f39cc0ca130 tx=0x7f39cc0994d0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 shutdown_connections 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d0fe0880 0x5617d0fe2ca0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fbfde0 0x5617d0fbd290 unknown :-1 s=CLOSED pgs=561 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb5780 0x5617d0fbf860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/194101369 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fbcba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 >> 172.21.15.90:0/194101369 conn(0x5617d0f146a0 msgr2=0x5617d0fafc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39b9ffb700 0 client.16248 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 shutdown_connections 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/194101369 wait complete. 2024-04-06T14:10:29.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.605+0000 7f39e4b6cb80 20 client.16248 trim_cache size 0 max 16384 2024-04-06T14:10:29.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 Processor -- start 2024-04-06T14:10:29.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 -- start start 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fb7690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fbfde0 0x5617d0fbc430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0fbcf20 con 0x5617d0fb5780 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0fbc970 con 0x5617d0fbfde0 2024-04-06T14:10:29.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.622+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d0fb21f0 con 0x5617d0fb60a0 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52438/0 (socket says 172.21.15.90:52438) 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 -- 172.21.15.90:0/2469785491 learned_addr learned my addr 172.21.15.90:0/2469785491 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d259c700 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fb7690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fbfde0 0x5617d0fbc430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 -- 172.21.15.90:0/2469785491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fbfde0 msgr2=0x5617d0fbc430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fbfde0 0x5617d0fbc430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 -- 172.21.15.90:0/2469785491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 msgr2=0x5617d0fb7690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fb7690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.623+0000 7f39d1d9b700 1 -- 172.21.15.90:0/2469785491 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5617d0d34b60 con 0x5617d0fb60a0 2024-04-06T14:10:29.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.624+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f39c8003730 tx=0x7f39c800b580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=65342a1ef2a3ba04 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.624+0000 7f39c17fa700 1 -- 172.21.15.90:0/2469785491 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c801d070 con 0x5617d0fb60a0 2024-04-06T14:10:29.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.624+0000 7f39c17fa700 1 -- 172.21.15.90:0/2469785491 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f39c801e040 con 0x5617d0fb60a0 2024-04-06T14:10:29.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.624+0000 7f39c17fa700 1 -- 172.21.15.90:0/2469785491 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39c8024480 con 0x5617d0fb60a0 2024-04-06T14:10:29.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/2469785491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 msgr2=0x5617d0fbd1a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f39c8003730 tx=0x7f39c800b580 comp rx=0 tx=0).stop 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/2469785491 shutdown_connections 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fbfde0 0x5617d0fbc430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fb7690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/2469785491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fb60a0 0x5617d0fbd1a0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/2469785491 >> 172.21.15.90:0/2469785491 conn(0x5617d0f146a0 msgr2=0x5617d0fe7600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/2469785491 shutdown_connections 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/2469785491 wait complete. 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- start start 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 Processor -- start 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fc4a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fc4f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc54c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d1033750 con 0x5617d0fb5780 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d1033890 con 0x5617d0fb60a0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617d1033a50 con 0x5617d0fbfde0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39e4b6cb80 10 client.0 _ll_register_callbacks cb 0x5617d0eac470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fc4a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fc4a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47398/0 (socket says 172.21.15.90:47398) 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39d259c700 1 -- 172.21.15.90:0/3038833960 learned_addr learned my addr 172.21.15.90:0/3038833960 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fc4f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.625+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3038833960 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 msgr2=0x5617d0fc4f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fc4f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3038833960 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 msgr2=0x5617d0fc4a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fc4a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 -- 172.21.15.90:0/3038833960 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f39d4114ae0 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d1d9b700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fc4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39d2d9d700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc54c0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f39cc00be90 tx=0x7f39cc00f420 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1de0bebea4934083 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39b9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f39cc020db0 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.626+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f39cc098fe0 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f39c4003b40 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f39cc09d920 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 10 client.25882 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5617d0eccea0 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f39cc07f030 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f39cc09ac20 con 0x5617d0fbfde0 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 10 client.25882 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 10 client.25882 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:10:29.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 20 client.25882 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d1034da0 0x5617d10371c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 4 client.25882 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5617d0d55cf0 con 0x5617d1034da0 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.627+0000 7f39d259c700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d1034da0 0x5617d10371c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39d259c700 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d1034da0 0x5617d10371c0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f3a31ee965fc380b server_cookie=a97baf8bc897dbdc in_seq=0 out_seq=0 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39b9ffb700 10 client.25882 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39b9ffb700 1 -- 172.21.15.90:0/3038833960 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5617d0d55cf0 con 0x5617d1034da0 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39b9ffb700 10 client.25882 handle_command_reply: tid=0 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 client.25882 shutdown 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 msgr2=0x5617d0fc54c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc54c0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f39cc00be90 tx=0x7f39cc00f420 comp rx=0 tx=0).stop 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 shutdown_connections 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5617d1034da0 0x5617d10371c0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617d0fbfde0 0x5617d0fc54c0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617d0fb5780 0x5617d0fc4a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 --2- 172.21.15.90:0/3038833960 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617d0fb60a0 0x5617d0fc4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 >> 172.21.15.90:0/3038833960 conn(0x5617d0f146a0 msgr2=0x5617d0fb7ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39b9ffb700 0 client.25882 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 shutdown_connections 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 1 -- 172.21.15.90:0/3038833960 wait complete. 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.628+0000 7f39e4b6cb80 20 client.25882 trim_cache size 0 max 16384 2024-04-06T14:10:29.633 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 msgr2=0x7f39bc0a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 0x7f39bc0a3920 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f39c4005950 tx=0x7f39c40058e0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 msgr2=0x7f39d419d6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 secure :-1 s=READY pgs=560 cs=0 l=1 rev1=1 crypto rx=0x7f39cc00b780 tx=0x7f39cc00ba90 comp rx=0 tx=0).stop 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 shutdown_connections 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f39bc0a1460 0x7f39bc0a3920 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f39d410d450 0x7f39d419d6d0 unknown :-1 s=CLOSED pgs=560 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f39d4107390 0x7f39d41a35a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 --2- 172.21.15.90:0/3769981742 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39d4107d50 0x7f39d41a3ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 >> 172.21.15.90:0/3769981742 conn(0x7f39d4076ba0 msgr2=0x7f39d4077f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 shutdown_connections 2024-04-06T14:10:29.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.629+0000 7f39d89b8700 1 -- 172.21.15.90:0/3769981742 wait complete. 2024-04-06T14:10:29.645 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:10:29.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:10:29.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:29 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2662509720' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:10:29.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.823+0000 7f9ecec83700 1 -- 172.21.15.90:0/2285147771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8100820 msgr2=0x7f9ec8104860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.823+0000 7f9ecec83700 1 --2- 172.21.15.90:0/2285147771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8100820 0x7f9ec8104860 secure :-1 s=READY pgs=562 cs=0 l=1 rev1=1 crypto rx=0x7f9eb4009990 tx=0x7f9eb4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 -- 172.21.15.90:0/2285147771 shutdown_connections 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 --2- 172.21.15.90:0/2285147771 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec8104f90 0x7f9ec810cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 --2- 172.21.15.90:0/2285147771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 0x7f9ec8100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 --2- 172.21.15.90:0/2285147771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8100820 0x7f9ec8104860 unknown :-1 s=CLOSED pgs=562 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 -- 172.21.15.90:0/2285147771 >> 172.21.15.90:0/2285147771 conn(0x7f9ec8076e10 msgr2=0x7f9ec8077220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 -- 172.21.15.90:0/2285147771 shutdown_connections 2024-04-06T14:10:29.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.824+0000 7f9ecec83700 1 -- 172.21.15.90:0/2285147771 wait complete. 2024-04-06T14:10:29.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 Processor -- start 2024-04-06T14:10:29.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 -- start start 2024-04-06T14:10:29.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 0x7f9ec80724d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 0x7f9ec8072a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ec806c080 con 0x7f9ec8104f90 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ec806c1f0 con 0x7f9ec80ffe70 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.825+0000 7f9ecec83700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ec81a61c0 con 0x7f9ec806b470 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52472/0 (socket says 172.21.15.90:52472) 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecca1f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 0x7f9ec80724d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 -- 172.21.15.90:0/3179306069 learned_addr learned my addr 172.21.15.90:0/3179306069 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 0x7f9ec8072a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 -- 172.21.15.90:0/3179306069 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 msgr2=0x7f9ec80724d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 0x7f9ec80724d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 -- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 msgr2=0x7f9ec8072a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 0x7f9ec8072a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 -- 172.21.15.90:0/3179306069 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9eb40095f0 con 0x7f9ec806b470 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 0x7f9ec8072a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecd220700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000efc0 tx=0x7f9ec000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ec5ffb700 1 -- 172.21.15.90:0/3179306069 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec0018070 con 0x7f9ec806b470 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ec81a63e0 con 0x7f9ec806b470 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ec5ffb700 1 -- 172.21.15.90:0/3179306069 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ec000f040 con 0x7f9ec806b470 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ec5ffb700 1 -- 172.21.15.90:0/3179306069 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec000ccc0 con 0x7f9ec806b470 2024-04-06T14:10:29.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.826+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ec81a68a0 con 0x7f9ec806b470 2024-04-06T14:10:29.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.829+0000 7f9ec5ffb700 1 -- 172.21.15.90:0/3179306069 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9ec0003a00 con 0x7f9ec806b470 2024-04-06T14:10:29.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.829+0000 7f9ec5ffb700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 0x7f9eb80a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.829+0000 7f9ec5ffb700 1 -- 172.21.15.90:0/3179306069 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ec00cc7e0 con 0x7f9ec806b470 2024-04-06T14:10:29.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.829+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 0x7f9eb80a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 Processor -- start 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 -- start start 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 0x7f9eb80a3750 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f9ebc009c60 tx=0x7f9ebc009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a9025b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a90260a0 0x5591a902f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8f7d990 con 0x5591a9025780 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8f7b350 con 0x5591a902fde0 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9edae37b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8f76c40 con 0x5591a90260a0 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecca1f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a9025b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecca1f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a9025b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47412/0 (socket says 172.21.15.90:47412) 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecca1f700 1 -- 172.21.15.90:0/1129060887 learned_addr learned my addr 172.21.15.90:0/1129060887 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.830+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48560/0 (socket says 172.21.15.90:48560) 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 -- 172.21.15.90:0/1129060887 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a90260a0 msgr2=0x5591a902f8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a90260a0 0x5591a902f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 -- 172.21.15.90:0/1129060887 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 msgr2=0x5591a9025b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a9025b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 -- 172.21.15.90:0/1129060887 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5591a8df83b0 con 0x5591a902fde0 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9ec00c9f00 tx=0x7f9ec00993f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e67e3c5839623800 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9eb2ffd700 1 -- 172.21.15.90:0/1129060887 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec00998b0 con 0x5591a902fde0 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9eb2ffd700 1 -- 172.21.15.90:0/1129060887 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ec009d6e0 con 0x5591a902fde0 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.831+0000 7f9eb2ffd700 1 -- 172.21.15.90:0/1129060887 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec001b320 con 0x5591a902fde0 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 -- 172.21.15.90:0/1129060887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 msgr2=0x5591a90321d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9ec00c9f00 tx=0x7f9ec00993f0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 -- 172.21.15.90:0/1129060887 shutdown_connections 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902fde0 0x5591a90321d0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a9025b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1129060887 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a90260a0 0x5591a902f8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.832+0000 7f9edae37b80 1 -- 172.21.15.90:0/1129060887 >> 172.21.15.90:0/1129060887 conn(0x5591a8f846a0 msgr2=0x5591a9020a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.833+0000 7f9edae37b80 1 -- 172.21.15.90:0/1129060887 shutdown_connections 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.833+0000 7f9edae37b80 1 -- 172.21.15.90:0/1129060887 wait complete. 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.833+0000 7f9edae37b80 1 -- start start 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.833+0000 7f9edae37b80 1 Processor -- start 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a902cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902d310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8f12d80 con 0x5591a902d850 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8f12a20 con 0x5591a9025780 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a8dfa6a0 con 0x5591a902fde0 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9edae37b80 10 client.0 _ll_register_callbacks cb 0x5591a8f1c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47426/0 (socket says 172.21.15.90:47426) 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9ecd220700 1 -- 172.21.15.90:0/1644571049 learned_addr learned my addr 172.21.15.90:0/1644571049 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.834+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a902cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 -- 172.21.15.90:0/1644571049 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 msgr2=0x5591a902d310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902d310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 -- 172.21.15.90:0/1644571049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 msgr2=0x5591a903bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 -- 172.21.15.90:0/1644571049 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ec81a68a0 con 0x5591a9025780 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a902cdd0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f9ebc003900 tx=0x7f9ebc0047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c262927721749347 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ea7fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ebc022070 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ebc01d6e0 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5591a8f37380 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ebc01d8a0 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9edae37b80 10 client.25979 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5591a8da4b60 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.835+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ebc033080 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ebc027ac0 con 0x5591a9025780 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 10 client.25979 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 10 client.25979 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 20 client.25979 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a903ff80 0x5591a90423a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 4 client.25979 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5591a8f6f700 con 0x5591a903ff80 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a903ff80 0x5591a90423a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9ea7fff700 10 client.25979 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.836+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a903ff80 0x5591a90423a0 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5f499bbe35faa75 server_cookie=f215715a7337de1d in_seq=0 out_seq=0 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1644571049 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5591a8f6f700 con 0x5591a903ff80 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9ea7fff700 10 client.25979 handle_command_reply: tid=0 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 client.25979 shutdown 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 msgr2=0x5591a902cdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a902cdd0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f9ebc003900 tx=0x7f9ebc0047c0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 shutdown_connections 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902d850 0x5591a903bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a902cdd0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a903ff80 0x5591a90423a0 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1644571049 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 >> 172.21.15.90:0/1644571049 conn(0x5591a8f846a0 msgr2=0x5591a901fc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9ea7fff700 0 client.25979 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:29.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 shutdown_connections 2024-04-06T14:10:29.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.837+0000 7f9edae37b80 1 -- 172.21.15.90:0/1644571049 wait complete. 2024-04-06T14:10:29.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.839+0000 7f9edae37b80 20 client.25979 trim_cache size 0 max 16384 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 Processor -- start 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 -- start start 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a90203a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902a050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a9022250 con 0x5591a9025780 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a9026a40 con 0x5591a902d850 2024-04-06T14:10:29.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9edae37b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a9021840 con 0x5591a902fde0 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9ec7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9ec7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48582/0 (socket says 172.21.15.90:48582) 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9ec7fff700 1 -- 172.21.15.90:0/659620566 learned_addr learned my addr 172.21.15.90:0/659620566 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a90203a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.857+0000 7f9ecd220700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902a050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 -- 172.21.15.90:0/659620566 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 msgr2=0x5591a90208e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 -- 172.21.15.90:0/659620566 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 msgr2=0x5591a90203a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a90203a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 -- 172.21.15.90:0/659620566 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5591a8f3cea0 con 0x5591a902fde0 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:29.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9ecd220700 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902a050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000c7a0 tx=0x7f9ec0009e50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d272304343b7e65e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9eb37fe700 1 -- 172.21.15.90:0/659620566 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec00b7070 con 0x5591a902fde0 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9eb37fe700 1 -- 172.21.15.90:0/659620566 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ec00998b0 con 0x5591a902fde0 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.858+0000 7f9eb37fe700 1 -- 172.21.15.90:0/659620566 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec0099b10 con 0x5591a902fde0 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- 172.21.15.90:0/659620566 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 msgr2=0x5591a902a050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902a050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000c7a0 tx=0x7f9ec0009e50 comp rx=0 tx=0).stop 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- 172.21.15.90:0/659620566 shutdown_connections 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902fde0 0x5591a902a050 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a9025780 0x5591a90203a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 --2- 172.21.15.90:0/659620566 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a902d850 0x5591a90208e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- 172.21.15.90:0/659620566 >> 172.21.15.90:0/659620566 conn(0x5591a8f846a0 msgr2=0x5591a902b6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- 172.21.15.90:0/659620566 shutdown_connections 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- 172.21.15.90:0/659620566 wait complete. 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 -- start start 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.859+0000 7f9edae37b80 1 Processor -- start 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a9034a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902d850 0x5591a9034f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a9092fb0 con 0x5591a902fde0 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a9093120 con 0x5591a9025780 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5591a90932e0 con 0x5591a902d850 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9edae37b80 10 client.0 _ll_register_callbacks cb 0x5591a8f1c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:29.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47444/0 (socket says 172.21.15.90:47444) 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 -- 172.21.15.90:0/1022268779 learned_addr learned my addr 172.21.15.90:0/1022268779 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ec7fff700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902d850 0x5591a9034f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 -- 172.21.15.90:0/1022268779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902d850 msgr2=0x5591a9034f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902d850 0x5591a9034f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 -- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 msgr2=0x5591a9034a40 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a9034a40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.860+0000 7f9ecd220700 1 -- 172.21.15.90:0/1022268779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ec810d380 con 0x5591a902fde0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9ecd220700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f9ec00b2340 tx=0x7f9ec0004d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6bb63d9564330472 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9ea7fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ec00b7070 con 0x5591a902fde0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ec00b9270 con 0x5591a902fde0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9ec009d6e0 con 0x5591a902fde0 2024-04-06T14:10:29.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ec0097ab0 con 0x5591a902fde0 2024-04-06T14:10:29.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9edae37b80 10 client.16272 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:29.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.861+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5591a8df83b0 con 0x5591a902fde0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.862+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ec009d6e0 con 0x5591a902fde0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.862+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ec00a1810 con 0x5591a902fde0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 10 client.16272 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 10 client.16272 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 20 client.16272 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a90945b0 0x5591a90969a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 4 client.16272 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5591a8dc5cf0 con 0x5591a90945b0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a90945b0 0x5591a90969a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9ecca1f700 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a90945b0 0x5591a90969a0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=773801c69f1ad969 server_cookie=dee510f1a5e63114 in_seq=0 out_seq=0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9ea7fff700 10 client.16272 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9ea7fff700 1 -- 172.21.15.90:0/1022268779 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5591a8dc5cf0 con 0x5591a90945b0 2024-04-06T14:10:29.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.863+0000 7f9ea7fff700 10 client.16272 handle_command_reply: tid=0 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 client.16272 shutdown 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 msgr2=0x5591a9035500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 secure :-1 s=READY pgs=563 cs=0 l=1 rev1=1 crypto rx=0x7f9ec00b2340 tx=0x7f9ec0004d30 comp rx=0 tx=0).stop 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 shutdown_connections 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5591a902fde0 0x5591a9035500 unknown :-1 s=CLOSED pgs=563 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5591a9025780 0x5591a9034a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5591a90945b0 0x5591a90969a0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 --2- 172.21.15.90:0/1022268779 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5591a902d850 0x5591a9034f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9ea7fff700 0 client.16272 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.864+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 >> 172.21.15.90:0/1022268779 conn(0x5591a8f846a0 msgr2=0x5591a9024770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 shutdown_connections 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9edae37b80 1 -- 172.21.15.90:0/1022268779 wait complete. 2024-04-06T14:10:29.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9edae37b80 20 client.16272 trim_cache size 0 max 16384 2024-04-06T14:10:29.867 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 msgr2=0x7f9eb80a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 0x7f9eb80a3750 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f9ebc009c60 tx=0x7f9ebc009260 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 msgr2=0x7f9ec806b8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f9ec000efc0 tx=0x7f9ec000c3a0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 shutdown_connections 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9eb80a1290 0x7f9eb80a3750 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ec806b470 0x7f9ec806b8f0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ec80ffe70 0x7f9ec80724d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 --2- 172.21.15.90:0/3179306069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ec8104f90 0x7f9ec8072a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:29.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.865+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 >> 172.21.15.90:0/3179306069 conn(0x7f9ec8076e10 msgr2=0x7f9ec80fde90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:29.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.866+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 shutdown_connections 2024-04-06T14:10:29.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:29.866+0000 7f9ecec83700 1 -- 172.21.15.90:0/3179306069 wait complete. 2024-04-06T14:10:29.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:29 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2662509720' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:10:29.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:29 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2662509720' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:10:30.882 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:10:30.882 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:10:30.882 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:10:30.919 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:30 smithi090 ceph-mon[43420]: pgmap v156: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 37 MiB/s wr, 2.12k op/s 2024-04-06T14:10:30.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:30 smithi105 ceph-mon[46969]: pgmap v156: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 37 MiB/s wr, 2.12k op/s 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 -- 172.21.15.90:0/3466760979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 msgr2=0x7ff4401013d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 --2- 172.21.15.90:0/3466760979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff4401013d0 secure :-1 s=READY pgs=564 cs=0 l=1 rev1=1 crypto rx=0x7ff430009990 tx=0x7ff430009ca0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 -- 172.21.15.90:0/3466760979 shutdown_connections 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 --2- 172.21.15.90:0/3466760979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 0x7ff44010dc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 --2- 172.21.15.90:0/3466760979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff4401013d0 unknown :-1 s=CLOSED pgs=564 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 --2- 172.21.15.90:0/3466760979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff4401019a0 0x7ff4401059e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 -- 172.21.15.90:0/3466760979 >> 172.21.15.90:0/3466760979 conn(0x7ff4400fc890 msgr2=0x7ff4400fecb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 -- 172.21.15.90:0/3466760979 shutdown_connections 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.058+0000 7ff44697c700 1 -- 172.21.15.90:0/3466760979 wait complete. 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 Processor -- start 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 -- start start 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 0x7ff4401977b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff44019bb50 0x7ff44019bfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff44019c5d0 con 0x7ff440100ff0 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff44019c740 con 0x7ff44019bb50 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff44697c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff44019c900 con 0x7ff440106110 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff43f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 0x7ff4401977b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47454/0 (socket says 172.21.15.90:47454) 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff43ffff700 1 -- 172.21.15.90:0/903615307 learned_addr learned my addr 172.21.15.90:0/903615307 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.059+0000 7ff444f19700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff44019bb50 0x7ff44019bfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 -- 172.21.15.90:0/903615307 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 msgr2=0x7ff4401977b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 0x7ff4401977b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 -- 172.21.15.90:0/903615307 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff44019bb50 msgr2=0x7ff44019bfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff44019bb50 0x7ff44019bfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 -- 172.21.15.90:0/903615307 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff434009520 con 0x7ff440100ff0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43ffff700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7ff430004760 tx=0x7ff430004790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43d7fa700 1 -- 172.21.15.90:0/903615307 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff43001d070 con 0x7ff440100ff0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43d7fa700 1 -- 172.21.15.90:0/903615307 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff430022470 con 0x7ff440100ff0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff43d7fa700 1 -- 172.21.15.90:0/903615307 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff430017650 con 0x7ff440100ff0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4300095f0 con 0x7ff440100ff0 2024-04-06T14:10:31.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.060+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4401a4090 con 0x7ff440100ff0 2024-04-06T14:10:31.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.064+0000 7ff452b30b80 1 Processor -- start 2024-04-06T14:10:31.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.064+0000 7ff43d7fa700 1 -- 172.21.15.90:0/903615307 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff4300225e0 con 0x7ff440100ff0 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 -- start start 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f5dcbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f5e6920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f534a10 con 0x56337f5dd120 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5323d0 con 0x56337f5e6e60 2024-04-06T14:10:31.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff452b30b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f52dcc0 con 0x56337f5dc800 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48624/0 (socket says 172.21.15.90:48624) 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 -- 172.21.15.90:0/1070190916 learned_addr learned my addr 172.21.15.90:0/1070190916 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff444f19700 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f5dcbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 -- 172.21.15.90:0/1070190916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 msgr2=0x56337f5dcbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f5dcbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 -- 172.21.15.90:0/1070190916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 msgr2=0x56337f5e6920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f5e6920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43f7fe700 1 -- 172.21.15.90:0/1070190916 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56337f3af430 con 0x56337f5e6e60 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.065+0000 7ff43d7fa700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 0x7ff4280a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff444f19700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 0x7ff4280a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7ff434009ef0 tx=0x7ff434015b30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=85c0b72b4772482b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff43d7fa700 1 -- 172.21.15.90:0/903615307 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff4300cdda0 con 0x7ff440100ff0 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff4267fc700 1 -- 172.21.15.90:0/1070190916 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4340154c0 con 0x56337f5e6e60 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff4267fc700 1 -- 172.21.15.90:0/1070190916 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff434004510 con 0x56337f5e6e60 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.066+0000 7ff4267fc700 1 -- 172.21.15.90:0/1070190916 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff434003880 con 0x56337f5e6e60 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- 172.21.15.90:0/1070190916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 msgr2=0x56337f5e9250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff444f19700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 0x7ff4280a3850 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7ff438007500 tx=0x7ff43800b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7ff434009ef0 tx=0x7ff434015b30 comp rx=0 tx=0).stop 2024-04-06T14:10:31.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- 172.21.15.90:0/1070190916 shutdown_connections 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5e9250 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f5dcbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- 172.21.15.90:0/1070190916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f5e6920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- 172.21.15.90:0/1070190916 >> 172.21.15.90:0/1070190916 conn(0x56337f53b720 msgr2=0x56337f5d7b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- 172.21.15.90:0/1070190916 shutdown_connections 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- 172.21.15.90:0/1070190916 wait complete. 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- start start 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 Processor -- start 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5e3bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5e4270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f4c9e00 con 0x56337f5dc800 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f4c9aa0 con 0x56337f5dd120 2024-04-06T14:10:31.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.067+0000 7ff452b30b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f3b1720 con 0x56337f5e6e60 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff452b30b80 10 client.0 _ll_register_callbacks cb 0x56337f4d34f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47466/0 (socket says 172.21.15.90:47466) 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff43ffff700 1 -- 172.21.15.90:0/2445495152 learned_addr learned my addr 172.21.15.90:0/2445495152 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5e4270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5e3bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 -- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 msgr2=0x56337f5e3bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5e3bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 -- 172.21.15.90:0/2445495152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 msgr2=0x56337f5e6860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 -- 172.21.15.90:0/2445495152 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff434009ac0 con 0x56337f5e6e60 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff43ffff700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff444f19700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5e4270 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7ff438007970 tx=0x7ff438004b40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a9dd295546172cff server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.068+0000 7ff41f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.069+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff43800e040 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.069+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff438015820 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.069+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56337f4ee400 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.070+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff4380159e0 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.070+0000 7ff452b30b80 10 client.25906 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.070+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56337f35bbe0 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.070+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56337f4ee400 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff438023c40 con 0x56337f5e6e60 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 10 client.25906 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 10 client.25906 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 20 client.25906 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f5f6b10 0x56337f5f8f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 4 client.25906 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56337f526780 con 0x56337f5f6b10 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff43ffff700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f5f6b10 0x56337f5f8f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff41f7fe700 10 client.25906 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.071+0000 7ff43ffff700 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f5f6b10 0x56337f5f8f30 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3bf409ad357f5573 server_cookie=c17a72aa16157a17 in_seq=0 out_seq=0 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff41f7fe700 1 -- 172.21.15.90:0/2445495152 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56337f526780 con 0x56337f5f6b10 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff41f7fe700 10 client.25906 handle_command_reply: tid=0 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 client.25906 shutdown 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 msgr2=0x56337f5e4270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5e4270 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7ff438007970 tx=0x7ff438004b40 comp rx=0 tx=0).stop 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 shutdown_connections 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f5f6b10 0x56337f5f8f30 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5e4270 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff41f7fe700 0 client.25906 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5e6860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 --2- 172.21.15.90:0/2445495152 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5e3bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 >> 172.21.15.90:0/2445495152 conn(0x56337f53b720 msgr2=0x56337f5d6ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 shutdown_connections 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.072+0000 7ff452b30b80 1 -- 172.21.15.90:0/2445495152 wait complete. 2024-04-06T14:10:31.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.073+0000 7ff452b30b80 20 client.25906 trim_cache size 0 max 16384 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 Processor -- start 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 -- start start 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5edae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5f2230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5ee020 con 0x56337f5dc800 2024-04-06T14:10:31.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5e4ea0 con 0x56337f5dd120 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff452b30b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5fd890 con 0x56337f5e6e60 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff43f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5edae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47468/0 (socket says 172.21.15.90:47468) 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff43f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5edae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48638/0 (socket says 172.21.15.90:48638) 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff43ffff700 1 -- 172.21.15.90:0/4217903773 learned_addr learned my addr 172.21.15.90:0/4217903773 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.093+0000 7ff444f19700 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5f2230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 -- 172.21.15.90:0/4217903773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 msgr2=0x56337f5f2230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5f2230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 -- 172.21.15.90:0/4217903773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 msgr2=0x56337f5edae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5edae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 -- 172.21.15.90:0/4217903773 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56337f5fdb60 con 0x56337f5dc800 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff43ffff700 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7ff43009aea0 tx=0x7ff430022cb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=66f6229736daf681 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff426ffd700 1 -- 172.21.15.90:0/4217903773 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4300b2070 con 0x56337f5dc800 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff426ffd700 1 -- 172.21.15.90:0/4217903773 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff43009aa50 con 0x56337f5dc800 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.094+0000 7ff426ffd700 1 -- 172.21.15.90:0/4217903773 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4300a3300 con 0x56337f5dc800 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- 172.21.15.90:0/4217903773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 msgr2=0x56337f5d7cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 secure :-1 s=READY pgs=566 cs=0 l=1 rev1=1 crypto rx=0x7ff43009aea0 tx=0x7ff430022cb0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- 172.21.15.90:0/4217903773 shutdown_connections 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5e6e60 0x56337f5f2230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dc800 0x56337f5d7cb0 unknown :-1 s=CLOSED pgs=566 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 --2- 172.21.15.90:0/4217903773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5dd120 0x56337f5edae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- 172.21.15.90:0/4217903773 >> 172.21.15.90:0/4217903773 conn(0x56337f53b720 msgr2=0x56337f5d75a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- 172.21.15.90:0/4217903773 shutdown_connections 2024-04-06T14:10:31.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- 172.21.15.90:0/4217903773 wait complete. 2024-04-06T14:10:31.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 -- start start 2024-04-06T14:10:31.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.095+0000 7ff452b30b80 1 Processor -- start 2024-04-06T14:10:31.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f644180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5ec9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5ed0a0 con 0x56337f5dd120 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5ed210 con 0x56337f5e6e60 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56337f5ed3d0 con 0x56337f5dc800 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.096+0000 7ff452b30b80 10 client.0 _ll_register_callbacks cb 0x56337f4d34f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52564/0 (socket says 172.21.15.90:52564) 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 -- 172.21.15.90:0/254466780 learned_addr learned my addr 172.21.15.90:0/254466780 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 -- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 msgr2=0x56337f5ec9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5ec9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 -- 172.21.15.90:0/254466780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 msgr2=0x56337f644180 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f644180 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 -- 172.21.15.90:0/254466780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4401a4090 con 0x56337f5dc800 2024-04-06T14:10:31.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff43ffff700 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7ff430097980 tx=0x7ff43009a920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=79e619fffbd34948 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff41f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.097+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff4300b2070 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff4300a3300 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff434004510 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff4300b9250 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 10 client.25912 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56337f4f3f20 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff434004510 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff4300a2e70 con 0x56337f5dc800 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 10 client.25912 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 10 client.25912 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 20 client.25912 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f64c710 0x56337f64eb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 4 client.25912 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.098+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x56337f37cd70 con 0x56337f64c710 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.099+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f64c710 0x56337f64eb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.099+0000 7ff41f7fe700 10 client.25912 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.099+0000 7ff43f7fe700 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f64c710 0x56337f64eb30 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e1092fa6c027577 server_cookie=332342ae535149b2 in_seq=0 out_seq=0 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.100+0000 7ff41f7fe700 1 -- 172.21.15.90:0/254466780 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x56337f37cd70 con 0x56337f64c710 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.100+0000 7ff41f7fe700 10 client.25912 handle_command_reply: tid=0 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.100+0000 7ff452b30b80 1 client.25912 shutdown 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 msgr2=0x56337f643c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7ff430097980 tx=0x7ff43009a920 comp rx=0 tx=0).stop 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 shutdown_connections 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56337f5e6e60 0x56337f5ec9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56337f64c710 0x56337f64eb30 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56337f5dc800 0x56337f643c40 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 --2- 172.21.15.90:0/254466780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56337f5dd120 0x56337f644180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff41f7fe700 0 client.25912 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 >> 172.21.15.90:0/254466780 conn(0x56337f53b720 msgr2=0x56337f5f2ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 shutdown_connections 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 1 -- 172.21.15.90:0/254466780 wait complete. 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff452b30b80 20 client.25912 trim_cache size 0 max 16384 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "daaa4042-ba89-43b7-9d91-1653db22f539", 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:10:31.104 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:10:31.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 msgr2=0x7ff4280a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 0x7ff4280a3850 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7ff438007500 tx=0x7ff43800b040 comp rx=0 tx=0).stop 2024-04-06T14:10:31.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 msgr2=0x7ff440197270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 secure :-1 s=READY pgs=565 cs=0 l=1 rev1=1 crypto rx=0x7ff430004760 tx=0x7ff430004790 comp rx=0 tx=0).stop 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 shutdown_connections 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff4280a1390 0x7ff4280a3850 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff44019bb50 0x7ff44019bfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff440100ff0 0x7ff440197270 unknown :-1 s=CLOSED pgs=565 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 --2- 172.21.15.90:0/903615307 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff440106110 0x7ff4401977b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.101+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 >> 172.21.15.90:0/903615307 conn(0x7ff4400fc890 msgr2=0x7ff4400fdc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.102+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 shutdown_connections 2024-04-06T14:10:31.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.102+0000 7ff44697c700 1 -- 172.21.15.90:0/903615307 wait complete. 2024-04-06T14:10:31.132 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:10:31.132 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:10:31.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:30 smithi134 ceph-mon[47549]: pgmap v156: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 37 MiB/s wr, 2.12k op/s 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 -- 172.21.15.90:0/1121179440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 msgr2=0x7ff8781011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1121179440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff8781011c0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7ff8680098f0 tx=0x7ff868009c00 comp rx=0 tx=0).stop 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 -- 172.21.15.90:0/1121179440 shutdown_connections 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1121179440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87810a250 0x7ff87810c6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1121179440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff8781011c0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1121179440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff878101790 0x7ff878109be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 -- 172.21.15.90:0/1121179440 >> 172.21.15.90:0/1121179440 conn(0x7ff878074b50 msgr2=0x7ff878074f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 -- 172.21.15.90:0/1121179440 shutdown_connections 2024-04-06T14:10:31.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.370+0000 7ff87e84f700 1 -- 172.21.15.90:0/1121179440 wait complete. 2024-04-06T14:10:31.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.371+0000 7ff87e84f700 1 Processor -- start 2024-04-06T14:10:31.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 -- start start 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff878199470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 0x7ff87819e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff87819e7d0 con 0x7ff87810a250 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff87819e940 con 0x7ff87819dd50 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.372+0000 7ff87e84f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff87819eb00 con 0x7ff878100de0 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47478/0 (socket says 172.21.15.90:47478) 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 -- 172.21.15.90:0/1477443803 learned_addr learned my addr 172.21.15.90:0/1477443803 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff87cdec700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 0x7ff87819e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff877fff700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff878199470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 -- 172.21.15.90:0/1477443803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 msgr2=0x7ff878199470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff878199470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 -- 172.21.15.90:0/1477443803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 msgr2=0x7ff87819e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 0x7ff87819e1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 -- 172.21.15.90:0/1477443803 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff868009520 con 0x7ff87810a250 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff87cdec700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 0x7ff87819e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.373+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7ff86c00e9f0 tx=0x7ff86c00ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.374+0000 7ff8757fa700 1 -- 172.21.15.90:0/1477443803 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff86c009690 con 0x7ff87810a250 2024-04-06T14:10:31.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.374+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff87806a830 con 0x7ff87810a250 2024-04-06T14:10:31.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.374+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff87806ad90 con 0x7ff87810a250 2024-04-06T14:10:31.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.374+0000 7ff8757fa700 1 -- 172.21.15.90:0/1477443803 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff86c00ce80 con 0x7ff87810a250 2024-04-06T14:10:31.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.374+0000 7ff8757fa700 1 -- 172.21.15.90:0/1477443803 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff86c018680 con 0x7ff87810a250 2024-04-06T14:10:31.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.379+0000 7ff88aa03b80 1 Processor -- start 2024-04-06T14:10:31.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.379+0000 7ff8757fa700 1 -- 172.21.15.90:0/1477443803 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff86c0187e0 con 0x7ff87810a250 2024-04-06T14:10:31.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.379+0000 7ff8757fa700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 0x7ff8600a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.379+0000 7ff8757fa700 1 -- 172.21.15.90:0/1477443803 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff86c014070 con 0x7ff87810a250 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 -- start start 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff87cdec700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 0x7ff8600a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df018a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4df041d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4de4f990 con 0x563b4def7780 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4de4d350 con 0x563b4df01de0 2024-04-06T14:10:31.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4de48c40 con 0x563b4def80a0 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff877fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff877fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47494/0 (socket says 172.21.15.90:47494) 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff877fff700 1 -- 172.21.15.90:0/2071811585 learned_addr learned my addr 172.21.15.90:0/2071811585 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.380+0000 7ff87cdec700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4df041d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff8777fe700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df018a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 -- 172.21.15.90:0/2071811585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 msgr2=0x563b4df018a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df018a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 -- 172.21.15.90:0/2071811585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 msgr2=0x563b4df041d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4df041d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 -- 172.21.15.90:0/2071811585 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563b4dcca3b0 con 0x563b4def7780 2024-04-06T14:10:31.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff8777fe700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df018a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:10:31.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff87cdec700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 0x7ff8600a3880 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7ff870007610 tx=0x7ff8700079d0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.381+0000 7ff877fff700 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7ff868016000 tx=0x7ff86800bfb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=21835126e27cce4d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.382+0000 7ff85e7fc700 1 -- 172.21.15.90:0/2071811585 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff86802e070 con 0x563b4def7780 2024-04-06T14:10:31.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.382+0000 7ff85e7fc700 1 -- 172.21.15.90:0/2071811585 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff86800f920 con 0x563b4def7780 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.382+0000 7ff85e7fc700 1 -- 172.21.15.90:0/2071811585 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8680052a0 con 0x563b4def7780 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 -- 172.21.15.90:0/2071811585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 msgr2=0x563b4def7b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 secure :-1 s=READY pgs=568 cs=0 l=1 rev1=1 crypto rx=0x7ff868016000 tx=0x7ff86800bfb0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 -- 172.21.15.90:0/2071811585 shutdown_connections 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4df041d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4def7b60 unknown :-1 s=CLOSED pgs=568 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/2071811585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df018a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.383+0000 7ff88aa03b80 1 -- 172.21.15.90:0/2071811585 >> 172.21.15.90:0/2071811585 conn(0x563b4de566a0 msgr2=0x563b4def2a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.385+0000 7ff88aa03b80 1 -- 172.21.15.90:0/2071811585 shutdown_connections 2024-04-06T14:10:31.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.385+0000 7ff88aa03b80 1 -- 172.21.15.90:0/2071811585 wait complete. 2024-04-06T14:10:31.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.385+0000 7ff88aa03b80 1 -- start start 2024-04-06T14:10:31.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 Processor -- start 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4df017a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4defeb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4deff140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4dde4d80 con 0x563b4def7780 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4dde4a20 con 0x563b4df01de0 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4dccc6a0 con 0x563b4def80a0 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff88aa03b80 10 client.0 _ll_register_callbacks cb 0x563b4ddee470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff877fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4df017a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff877fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4df017a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47506/0 (socket says 172.21.15.90:47506) 2024-04-06T14:10:31.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.386+0000 7ff877fff700 1 -- 172.21.15.90:0/3602823735 learned_addr learned my addr 172.21.15.90:0/3602823735 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4deff140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff8777fe700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4defeb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 -- 172.21.15.90:0/3602823735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 msgr2=0x563b4defeb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4defeb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 -- 172.21.15.90:0/3602823735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 msgr2=0x563b4df017a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4df017a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 -- 172.21.15.90:0/3602823735 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff87810d360 con 0x563b4df01de0 2024-04-06T14:10:31.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff87cdec700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4deff140 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x563b4def91f0 tx=0x7ff870004b40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9dca6fcc33bc641 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.387+0000 7ff8577fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:10:31.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.388+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff870021070 con 0x563b4df01de0 2024-04-06T14:10:31.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.388+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff870019b70 con 0x563b4df01de0 2024-04-06T14:10:31.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563b4de09380 con 0x563b4df01de0 2024-04-06T14:10:31.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff870019d30 con 0x563b4df01de0 2024-04-06T14:10:31.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 10 client.26006 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:31.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563b4dc76b60 con 0x563b4df01de0 2024-04-06T14:10:31.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563b4de09380 con 0x563b4df01de0 2024-04-06T14:10:31.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff87002f070 con 0x563b4df01de0 2024-04-06T14:10:31.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 10 client.26006 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 10 client.26006 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 20 client.26006 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df1a0d0 0x563b4df1c4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 4 client.26006 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.390+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563b4de41700 con 0x563b4df1a0d0 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.391+0000 7ff877fff700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df1a0d0 0x563b4df1c4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.391+0000 7ff877fff700 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df1a0d0 0x563b4df1c4f0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=923a9abfc2235d0a server_cookie=abc9c43967bee286 in_seq=0 out_seq=0 2024-04-06T14:10:31.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.391+0000 7ff8577fe700 10 client.26006 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff8577fe700 1 -- 172.21.15.90:0/3602823735 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563b4de41700 con 0x563b4df1a0d0 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff8577fe700 10 client.26006 handle_command_reply: tid=0 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 client.26006 shutdown 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 msgr2=0x563b4deff140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4deff140 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x563b4def91f0 tx=0x7ff870004b40 comp rx=0 tx=0).stop 2024-04-06T14:10:31.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 shutdown_connections 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df01de0 0x563b4deff140 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4def7780 0x563b4df017a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df1a0d0 0x563b4df1c4f0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/3602823735 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4defeb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff8577fe700 0 client.26006 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 >> 172.21.15.90:0/3602823735 conn(0x563b4de566a0 msgr2=0x563b4def1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.392+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 shutdown_connections 2024-04-06T14:10:31.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.395+0000 7ff88aa03b80 1 -- 172.21.15.90:0/3602823735 wait complete. 2024-04-06T14:10:31.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.396+0000 7ff88aa03b80 20 client.26006 trim_cache size 0 max 16384 2024-04-06T14:10:31.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 Processor -- start 2024-04-06T14:10:31.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 -- start start 2024-04-06T14:10:31.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 0x563b4df11320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 0x563b4def2520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4def2a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4defe950 con 0x563b4df01de0 2024-04-06T14:10:31.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4defeeb0 con 0x563b4def80a0 2024-04-06T14:10:31.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.416+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4def9220 con 0x563b4def7780 2024-04-06T14:10:31.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff877fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 0x563b4df11320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff877fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 0x563b4df11320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52600/0 (socket says 172.21.15.90:52600) 2024-04-06T14:10:31.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff877fff700 1 -- 172.21.15.90:0/1208938979 learned_addr learned my addr 172.21.15.90:0/1208938979 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff87cdec700 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4def2a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 0x563b4def2520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 -- 172.21.15.90:0/1208938979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 msgr2=0x563b4df11320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 0x563b4df11320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 -- 172.21.15.90:0/1208938979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 msgr2=0x563b4def2a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4def2a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 -- 172.21.15.90:0/1208938979 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563b4de0eea0 con 0x563b4def80a0 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.417+0000 7ff8777fe700 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 0x563b4def2520 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0cb480 tx=0x7ff86c09a9b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1f55322a274ca194 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff85effd700 1 -- 172.21.15.90:0/1208938979 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff86c09b2d0 con 0x563b4def80a0 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff85effd700 1 -- 172.21.15.90:0/1208938979 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff86c09a360 con 0x563b4def80a0 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff85effd700 1 -- 172.21.15.90:0/1208938979 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff86c096ec0 con 0x563b4def80a0 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 -- 172.21.15.90:0/1208938979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 msgr2=0x563b4def2520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 0x563b4def2520 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7ff86c0cb480 tx=0x7ff86c09a9b0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 -- 172.21.15.90:0/1208938979 shutdown_connections 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4def2a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def7780 0x563b4df11320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/1208938979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4def80a0 0x563b4def2520 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.418+0000 7ff88aa03b80 1 -- 172.21.15.90:0/1208938979 >> 172.21.15.90:0/1208938979 conn(0x563b4de566a0 msgr2=0x563b4defd400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.420+0000 7ff88aa03b80 1 -- 172.21.15.90:0/1208938979 shutdown_connections 2024-04-06T14:10:31.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.420+0000 7ff88aa03b80 1 -- 172.21.15.90:0/1208938979 wait complete. 2024-04-06T14:10:31.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 -- start start 2024-04-06T14:10:31.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 Processor -- start 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df0f9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4df0fee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4df6e4a0 con 0x563b4df01de0 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4df6e610 con 0x563b4df10420 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563b4df6e7d0 con 0x563b4def80a0 2024-04-06T14:10:31.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.421+0000 7ff88aa03b80 10 client.0 _ll_register_callbacks cb 0x563b4ddee470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:10:31.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff877fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df0f9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:48692/0 (socket says 172.21.15.90:48692) 2024-04-06T14:10:31.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff8777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4df0fee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 -- 172.21.15.90:0/398092545 learned_addr learned my addr 172.21.15.90:0/398092545 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 -- 172.21.15.90:0/398092545 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 msgr2=0x563b4df0f9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df0f9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 -- 172.21.15.90:0/398092545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 msgr2=0x563b4df0fee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4df0fee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 -- 172.21.15.90:0/398092545 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff87806ad90 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff87cdec700 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff870004a90 tx=0x7ff870004ac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f2acb35b05b7345d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.422+0000 7ff8577fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff870021050 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff870018490 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff86800f920 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff870018d90 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff88aa03b80 10 client.26012 fetch_fsmap learned FSMap version 24 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.423+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563b4dccecd0 con 0x563b4df10420 2024-04-06T14:10:31.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff86800f920 con 0x563b4df10420 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff870019400 con 0x563b4df10420 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 10 client.26012 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 10 client.26012 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 20 client.26012 populate_metadata read hostname 'smithi090' 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df04fb0 0x563b4df073d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 4 client.26012 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.424+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x563b4dc97cf0 con 0x563b4df04fb0 2024-04-06T14:10:31.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.425+0000 7ff877fff700 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df04fb0 0x563b4df073d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.425+0000 7ff877fff700 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df04fb0 0x563b4df073d0 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=82d0a45e9471446 server_cookie=c7a4ffbb2491fa54 in_seq=0 out_seq=0 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.425+0000 7ff8577fe700 10 client.26012 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff8577fe700 1 -- 172.21.15.90:0/398092545 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x563b4dc97cf0 con 0x563b4df04fb0 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff8577fe700 10 client.26012 handle_command_reply: tid=0 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 client.26012 shutdown 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:10:31.426 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: "daaa4042-ba89-43b7-9d91-1653db22f539": { 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "daaa4042-ba89-43b7-9d91-1653db22f539", 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:10:31.427 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 msgr2=0x563b4df6df10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7ff870004a90 tx=0x7ff870004ac0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 shutdown_connections 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563b4df10420 0x563b4df6df10 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563b4df04fb0 0x563b4df073d0 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563b4def80a0 0x563b4df0f9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 --2- 172.21.15.90:0/398092545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563b4df01de0 0x563b4df0fee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 >> 172.21.15.90:0/398092545 conn(0x563b4de566a0 msgr2=0x563b4defd340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff8577fe700 0 client.26012 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 shutdown_connections 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 1 -- 172.21.15.90:0/398092545 wait complete. 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.426+0000 7ff88aa03b80 20 client.26012 trim_cache size 0 max 16384 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.427+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 msgr2=0x7ff8600a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.427+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 0x7ff8600a3880 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7ff870007610 tx=0x7ff8700079d0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.427+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 msgr2=0x7ff8781999b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.427+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 secure :-1 s=READY pgs=567 cs=0 l=1 rev1=1 crypto rx=0x7ff86c00e9f0 tx=0x7ff86c00ed00 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 shutdown_connections 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff8600a13c0 0x7ff8600a3880 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff87819dd50 0x7ff87819e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff878100de0 0x7ff878199470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 --2- 172.21.15.90:0/1477443803 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff87810a250 0x7ff8781999b0 unknown :-1 s=CLOSED pgs=567 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:10:31.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.428+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 >> 172.21.15.90:0/1477443803 conn(0x7ff878074b50 msgr2=0x7ff8780fc270 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:10:31.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.429+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 shutdown_connections 2024-04-06T14:10:31.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:10:31.429+0000 7ff87e84f700 1 -- 172.21.15.90:0/1477443803 wait complete. 2024-04-06T14:10:31.448 INFO:tasks.cephfs.filesystem:scrub status for tag:daaa4042-ba89-43b7-9d91-1653db22f539 - {'path': '/', 'tag': 'daaa4042-ba89-43b7-9d91-1653db22f539', 'options': 'recursive,force'} 2024-04-06T14:10:31.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:31 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:10:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:31 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:31 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:10:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:31 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:10:31.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:31 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:31 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:10:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:31 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:31 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:10:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:31 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:10:32.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:31 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:32.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:31 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:10:32.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:31 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:32.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:31 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:10:32.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:31 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:10:32.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:31 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:10:32.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:32 smithi105 ceph-mon[46969]: pgmap v157: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 39 MiB/s wr, 2.20k op/s 2024-04-06T14:10:33.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:10:33.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:10:33.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:10:33.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 285.0 s, 2429.2 tps, lat 41.136 ms stddev 54.065 2024-04-06T14:10:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:32 smithi090 ceph-mon[43420]: pgmap v157: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 39 MiB/s wr, 2.20k op/s 2024-04-06T14:10:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:10:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:10:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:10:33.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:32 smithi134 ceph-mon[47549]: pgmap v157: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 39 MiB/s wr, 2.20k op/s 2024-04-06T14:10:33.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:10:33.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:10:33.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:10:35.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:34 smithi090 ceph-mon[43420]: pgmap v158: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 32 MiB/s wr, 1.98k op/s 2024-04-06T14:10:35.148 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:34 smithi134 ceph-mon[47549]: pgmap v158: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 32 MiB/s wr, 1.98k op/s 2024-04-06T14:10:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:34 smithi105 ceph-mon[46969]: pgmap v158: 97 pgs: 97 active+clean; 7.1 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 32 MiB/s wr, 1.98k op/s 2024-04-06T14:10:36.154 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:35 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:10:36.154 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:35 smithi134 ceph-mon[47549]: pgmap v159: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 127 KiB/s rd, 38 MiB/s wr, 2.27k op/s 2024-04-06T14:10:36.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:35 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:10:36.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:35 smithi105 ceph-mon[46969]: pgmap v159: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 127 KiB/s rd, 38 MiB/s wr, 2.27k op/s 2024-04-06T14:10:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:35 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:10:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:35 smithi090 ceph-mon[43420]: pgmap v159: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 127 KiB/s rd, 38 MiB/s wr, 2.27k op/s 2024-04-06T14:10:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 290.0 s, 2713.8 tps, lat 36.684 ms stddev 40.609 2024-04-06T14:10:38.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:38 smithi134 ceph-mon[47549]: pgmap v160: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 557 op/s 2024-04-06T14:10:39.001 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:38 smithi105 ceph-mon[46969]: pgmap v160: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 557 op/s 2024-04-06T14:10:39.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:38 smithi090 ceph-mon[43420]: pgmap v160: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 557 op/s 2024-04-06T14:10:40.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:40 smithi105 ceph-mon[46969]: pgmap v161: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 16 MiB/s wr, 694 op/s 2024-04-06T14:10:41.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:40 smithi090 ceph-mon[43420]: pgmap v161: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 16 MiB/s wr, 694 op/s 2024-04-06T14:10:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:40 smithi134 ceph-mon[47549]: pgmap v161: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 16 MiB/s wr, 694 op/s 2024-04-06T14:10:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 295.0 s, 2943.8 tps, lat 34.095 ms stddev 36.894 2024-04-06T14:10:43.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:42 smithi090 ceph-mon[43420]: pgmap v162: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:10:43.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:42 smithi134 ceph-mon[47549]: pgmap v162: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:10:43.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:42 smithi105 ceph-mon[46969]: pgmap v162: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 123 KiB/s rd, 15 MiB/s wr, 755 op/s 2024-04-06T14:10:45.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:44 smithi090 ceph-mon[43420]: pgmap v163: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 11 MiB/s wr, 591 op/s 2024-04-06T14:10:45.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:44 smithi134 ceph-mon[47549]: pgmap v163: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 11 MiB/s wr, 591 op/s 2024-04-06T14:10:45.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:44 smithi105 ceph-mon[46969]: pgmap v163: 97 pgs: 97 active+clean; 7.2 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 11 MiB/s wr, 591 op/s 2024-04-06T14:10:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:46 smithi090 ceph-mon[43420]: pgmap v164: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 17 MiB/s wr, 893 op/s 2024-04-06T14:10:47.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:46 smithi134 ceph-mon[47549]: pgmap v164: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 17 MiB/s wr, 893 op/s 2024-04-06T14:10:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:46 smithi105 ceph-mon[46969]: pgmap v164: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 17 MiB/s wr, 893 op/s 2024-04-06T14:10:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 300.0 s, 2583.8 tps, lat 38.719 ms stddev 41.001 2024-04-06T14:10:48.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:48 smithi105 ceph-mon[46969]: pgmap v165: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 11 MiB/s wr, 600 op/s 2024-04-06T14:10:49.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:48 smithi090 ceph-mon[43420]: pgmap v165: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 11 MiB/s wr, 600 op/s 2024-04-06T14:10:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:48 smithi134 ceph-mon[47549]: pgmap v165: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 11 MiB/s wr, 600 op/s 2024-04-06T14:10:51.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:50 smithi090 ceph-mon[43420]: pgmap v166: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 12 MiB/s wr, 665 op/s 2024-04-06T14:10:51.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:50 smithi134 ceph-mon[47549]: pgmap v166: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 12 MiB/s wr, 665 op/s 2024-04-06T14:10:51.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:50 smithi105 ceph-mon[46969]: pgmap v166: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 12 MiB/s wr, 665 op/s 2024-04-06T14:10:53.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 305.0 s, 2354.8 tps, lat 42.357 ms stddev 49.881 2024-04-06T14:10:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:52 smithi090 ceph-mon[43420]: pgmap v167: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 11 MiB/s wr, 751 op/s 2024-04-06T14:10:53.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:52 smithi134 ceph-mon[47549]: pgmap v167: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 11 MiB/s wr, 751 op/s 2024-04-06T14:10:53.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:52 smithi105 ceph-mon[46969]: pgmap v167: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 11 MiB/s wr, 751 op/s 2024-04-06T14:10:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:54 smithi090 ceph-mon[43420]: pgmap v168: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 9.2 MiB/s wr, 590 op/s 2024-04-06T14:10:55.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:54 smithi134 ceph-mon[47549]: pgmap v168: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 9.2 MiB/s wr, 590 op/s 2024-04-06T14:10:55.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:54 smithi105 ceph-mon[46969]: pgmap v168: 97 pgs: 97 active+clean; 7.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 9.2 MiB/s wr, 590 op/s 2024-04-06T14:10:57.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:56 smithi090 ceph-mon[43420]: pgmap v169: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 17 MiB/s wr, 1.00k op/s 2024-04-06T14:10:57.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:56 smithi134 ceph-mon[47549]: pgmap v169: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 17 MiB/s wr, 1.00k op/s 2024-04-06T14:10:57.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:56 smithi105 ceph-mon[46969]: pgmap v169: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 17 MiB/s wr, 1.00k op/s 2024-04-06T14:10:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 310.0 s, 1820.8 tps, lat 54.920 ms stddev 65.950 2024-04-06T14:10:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:10:58 smithi105 ceph-mon[46969]: pgmap v170: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 12 MiB/s wr, 702 op/s 2024-04-06T14:10:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:10:58 smithi090 ceph-mon[43420]: pgmap v170: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 12 MiB/s wr, 702 op/s 2024-04-06T14:10:59.154 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:10:58 smithi134 ceph-mon[47549]: pgmap v170: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 12 MiB/s wr, 702 op/s 2024-04-06T14:11:01.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:00 smithi134 ceph-mon[47549]: pgmap v171: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 15 MiB/s wr, 834 op/s 2024-04-06T14:11:01.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:00 smithi105 ceph-mon[46969]: pgmap v171: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 15 MiB/s wr, 834 op/s 2024-04-06T14:11:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:00 smithi090 ceph-mon[43420]: pgmap v171: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 15 MiB/s wr, 834 op/s 2024-04-06T14:11:01.448 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:11:01.448 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 -- 172.21.15.90:0/3372825232 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8113f00 msgr2=0x7f69c81162f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 --2- 172.21.15.90:0/3372825232 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8113f00 0x7f69c81162f0 secure :-1 s=READY pgs=569 cs=0 l=1 rev1=1 crypto rx=0x7f69c000b0d0 tx=0x7f69c000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 -- 172.21.15.90:0/3372825232 shutdown_connections 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 --2- 172.21.15.90:0/3372825232 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8113f00 0x7f69c81162f0 unknown :-1 s=CLOSED pgs=569 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 --2- 172.21.15.90:0/3372825232 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8107db0 0x7f69c8108190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 --2- 172.21.15.90:0/3372825232 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 0x7f69c81139c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 -- 172.21.15.90:0/3372825232 >> 172.21.15.90:0/3372825232 conn(0x7f69c806be00 msgr2=0x7f69c806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 -- 172.21.15.90:0/3372825232 shutdown_connections 2024-04-06T14:11:01.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.621+0000 7f69cd53b700 1 -- 172.21.15.90:0/3372825232 wait complete. 2024-04-06T14:11:01.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 Processor -- start 2024-04-06T14:11:01.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 -- start start 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 0x7f69c81a5d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 0x7f69c819f920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c819ff80 con 0x7f69c8107db0 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c81a00f0 con 0x7f69c8113f00 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69cd53b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c81a02b0 con 0x7f69c81086d0 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47788/0 (socket says 172.21.15.90:47788) 2024-04-06T14:11:01.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 -- 172.21.15.90:0/1856630250 learned_addr learned my addr 172.21.15.90:0/1856630250 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c77fe700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 0x7f69c819f920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 0x7f69c81a5d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 -- 172.21.15.90:0/1856630250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 msgr2=0x7f69c81a5d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 0x7f69c81a5d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 -- 172.21.15.90:0/1856630250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 msgr2=0x7f69c819f920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 0x7f69c819f920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c6ffd700 1 -- 172.21.15.90:0/1856630250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f69c0009d00 con 0x7f69c8107db0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.622+0000 7f69c77fe700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 0x7f69c819f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f69b800e9a0 tx=0x7f69b800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69affff700 1 -- 172.21.15.90:0/1856630250 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69b8009790 con 0x7f69c8107db0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69affff700 1 -- 172.21.15.90:0/1856630250 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69b8010430 con 0x7f69c8107db0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69c81a0590 con 0x7f69c8107db0 2024-04-06T14:11:01.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69affff700 1 -- 172.21.15.90:0/1856630250 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69b8018680 con 0x7f69c8107db0 2024-04-06T14:11:01.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.623+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69c81ac7e0 con 0x7f69c8107db0 2024-04-06T14:11:01.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.625+0000 7f69affff700 1 -- 172.21.15.90:0/1856630250 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f69b80105a0 con 0x7f69c8107db0 2024-04-06T14:11:01.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.626+0000 7f69affff700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 0x7f69b00a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.626+0000 7f69affff700 1 -- 172.21.15.90:0/1856630250 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f69b8014070 con 0x7f69c8107db0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.626+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 0x7f69b00a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 0x7f69b00a3850 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f69c81a0ae0 tx=0x7f69bc009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 Processor -- start 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 -- start start 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb7b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51e0f990 con 0x55cf51eb80a0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51e0d350 con 0x55cf51ec1de0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69d96efb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51e08c40 con 0x55cf51eb7780 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47794/0 (socket says 172.21.15.90:47794) 2024-04-06T14:11:01.630 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 -- 172.21.15.90:0/2427897855 learned_addr learned my addr 172.21.15.90:0/2427897855 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c77fe700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb7b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 -- 172.21.15.90:0/2427897855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 msgr2=0x55cf51eb7b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb7b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 -- 172.21.15.90:0/2427897855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 msgr2=0x55cf51ec41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c67fc700 1 -- 172.21.15.90:0/2427897855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf51c8a3b0 con 0x55cf51eb80a0 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c77fe700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.627+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb7b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:01.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.628+0000 7f69c67fc700 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7f69bc009fd0 tx=0x7f69bc011000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95d48633901d8a9e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.629+0000 7f69ad7fa700 1 -- 172.21.15.90:0/2427897855 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc00ee90 con 0x55cf51eb80a0 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.629+0000 7f69ad7fa700 1 -- 172.21.15.90:0/2427897855 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69bc015940 con 0x55cf51eb80a0 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.629+0000 7f69ad7fa700 1 -- 172.21.15.90:0/2427897855 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc0139d0 con 0x55cf51eb80a0 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.630+0000 7f69d96efb80 1 -- 172.21.15.90:0/2427897855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 msgr2=0x55cf51ec18a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.630+0000 7f69d96efb80 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 secure :-1 s=READY pgs=571 cs=0 l=1 rev1=1 crypto rx=0x7f69bc009fd0 tx=0x7f69bc011000 comp rx=0 tx=0).stop 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.631+0000 7f69d96efb80 1 -- 172.21.15.90:0/2427897855 shutdown_connections 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.631+0000 7f69d96efb80 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.631+0000 7f69d96efb80 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb7b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.631+0000 7f69d96efb80 1 --2- 172.21.15.90:0/2427897855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ec18a0 unknown :-1 s=CLOSED pgs=571 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.631+0000 7f69d96efb80 1 -- 172.21.15.90:0/2427897855 >> 172.21.15.90:0/2427897855 conn(0x55cf51e166a0 msgr2=0x55cf51eb2a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- 172.21.15.90:0/2427897855 shutdown_connections 2024-04-06T14:11:01.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- 172.21.15.90:0/2427897855 wait complete. 2024-04-06T14:11:01.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- start start 2024-04-06T14:11:01.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 Processor -- start 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51ebec20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51eb80a0 0x55cf51ebf160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51da4d80 con 0x55cf51ec1de0 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51da4a20 con 0x55cf51eb80a0 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51c8c6a0 con 0x55cf51eb7780 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.633+0000 7f69d96efb80 10 client.0 _ll_register_callbacks cb 0x55cf51dae470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:01.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47806/0 (socket says 172.21.15.90:47806) 2024-04-06T14:11:01.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c77fe700 1 -- 172.21.15.90:0/966954187 learned_addr learned my addr 172.21.15.90:0/966954187 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51ebec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c67fc700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51eb80a0 0x55cf51ebf160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 -- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51eb80a0 msgr2=0x55cf51ebf160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51eb80a0 0x55cf51ebf160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 -- 172.21.15.90:0/966954187 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 msgr2=0x55cf51ebf6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c6ffd700 1 -- 172.21.15.90:0/966954187 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69c8116cc0 con 0x55cf51eb7780 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.634+0000 7f69c77fe700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51ebec20 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f69b80ca580 tx=0x7f69b80caae0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cf77036cdf2b29ce server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69aaff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:01.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69b8099630 con 0x55cf51eb7780 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f69b80a23c0 con 0x55cf51eb7780 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cf51dc9380 con 0x55cf51eb7780 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f69b80a1f30 con 0x55cf51eb7780 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69d96efb80 10 client.25936 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f69bc00e6c0 con 0x55cf51eb7780 2024-04-06T14:11:01.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.635+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55cf51dc9380 con 0x55cf51eb7780 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f69b8095d30 con 0x55cf51eb7780 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 10 client.25936 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 10 client.25936 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 20 client.25936 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ec89a0 0x55cf51ecad90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 4 client.25936 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cf51e01700 con 0x55cf51ec89a0 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.636+0000 7f69c67fc700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ec89a0 0x55cf51ecad90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.637+0000 7f69c67fc700 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ec89a0 0x55cf51ecad90 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cdda1d6e1e1f2f70 server_cookie=f2b952b9feeae2e0 in_seq=0 out_seq=0 2024-04-06T14:11:01.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.637+0000 7f69aaff5700 10 client.25936 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.637+0000 7f69aaff5700 1 -- 172.21.15.90:0/966954187 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cf51e01700 con 0x55cf51ec89a0 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.637+0000 7f69aaff5700 10 client.25936 handle_command_reply: tid=0 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.637+0000 7f69d96efb80 1 client.25936 shutdown 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 msgr2=0x55cf51ebec20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51ebec20 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f69b80ca580 tx=0x7f69b80caae0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 shutdown_connections 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51ebf6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ec89a0 0x55cf51ecad90 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51ebec20 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69aaff5700 0 client.25936 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 --2- 172.21.15.90:0/966954187 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51eb80a0 0x55cf51ebf160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 >> 172.21.15.90:0/966954187 conn(0x55cf51e166a0 msgr2=0x55cf51eb1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 shutdown_connections 2024-04-06T14:11:01.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 1 -- 172.21.15.90:0/966954187 wait complete. 2024-04-06T14:11:01.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.638+0000 7f69d96efb80 20 client.25936 trim_cache size 0 max 16384 2024-04-06T14:11:01.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.670+0000 7f69d96efb80 1 Processor -- start 2024-04-06T14:11:01.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 -- start start 2024-04-06T14:11:01.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb3dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ecf6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec0d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ecfc00 con 0x55cf51eb80a0 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ec1240 con 0x55cf51ec1de0 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69d96efb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ebd130 con 0x55cf51eb7780 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec0d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec0d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43732/0 (socket says 172.21.15.90:43732) 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69c77fe700 1 -- 172.21.15.90:0/3231449151 learned_addr learned my addr 172.21.15.90:0/3231449151 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.671+0000 7f69c67fc700 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ecf6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 -- 172.21.15.90:0/3231449151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 msgr2=0x55cf51eb3dc0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:01.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb3dc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 -- 172.21.15.90:0/3231449151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 msgr2=0x55cf51ec0d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec0d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 -- 172.21.15.90:0/3231449151 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf51c36b60 con 0x55cf51eb80a0 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69c67fc700 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ecf6c0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f69bc00ece0 tx=0x7f69bc0134f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=413fac59f19ae7e5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69adffb700 1 -- 172.21.15.90:0/3231449151 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc02e040 con 0x55cf51eb80a0 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69adffb700 1 -- 172.21.15.90:0/3231449151 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69bc00ee90 con 0x55cf51eb80a0 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.672+0000 7f69adffb700 1 -- 172.21.15.90:0/3231449151 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc032730 con 0x55cf51eb80a0 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.673+0000 7f69d96efb80 1 -- 172.21.15.90:0/3231449151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 msgr2=0x55cf51ecf6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.673+0000 7f69d96efb80 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ecf6c0 secure :-1 s=READY pgs=572 cs=0 l=1 rev1=1 crypto rx=0x7f69bc00ece0 tx=0x7f69bc0134f0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.674+0000 7f69d96efb80 1 -- 172.21.15.90:0/3231449151 shutdown_connections 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.674+0000 7f69d96efb80 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ec1de0 0x55cf51ec0d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.674+0000 7f69d96efb80 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb7780 0x55cf51eb3dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.674+0000 7f69d96efb80 1 --2- 172.21.15.90:0/3231449151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51eb80a0 0x55cf51ecf6c0 unknown :-1 s=CLOSED pgs=572 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.674+0000 7f69d96efb80 1 -- 172.21.15.90:0/3231449151 >> 172.21.15.90:0/3231449151 conn(0x55cf51e166a0 msgr2=0x55cf51eeb350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.675+0000 7f69d96efb80 1 -- 172.21.15.90:0/3231449151 shutdown_connections 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.675+0000 7f69d96efb80 1 -- 172.21.15.90:0/3231449151 wait complete. 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.675+0000 7f69d96efb80 1 -- start start 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.675+0000 7f69d96efb80 1 Processor -- start 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb80a0 0x55cf51edb150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 0x55cf51ed6dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ed7300 con 0x55cf51ec1de0 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ed7470 con 0x55cf51ed6940 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf51ed7630 con 0x55cf51eb80a0 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.678+0000 7f69d96efb80 10 client.0 _ll_register_callbacks cb 0x55cf51dae470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 0x55cf51ed6dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 0x55cf51ed6dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43740/0 (socket says 172.21.15.90:43740) 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47816/0 (socket says 172.21.15.90:47816) 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c77fe700 1 -- 172.21.15.90:0/1211485186 learned_addr learned my addr 172.21.15.90:0/1211485186 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 -- 172.21.15.90:0/1211485186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb80a0 msgr2=0x55cf51edb150 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb80a0 0x55cf51edb150 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 -- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 msgr2=0x55cf51ed6dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 0x55cf51ed6dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 -- 172.21.15.90:0/1211485186 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55cf51c8a3b0 con 0x55cf51ec1de0 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69c67fc700 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x55cf51ec6180 tx=0x7f69bc015cd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4cf2feb016930467 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.679+0000 7f69aaff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.681+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc02e040 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.681+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f69bc0325a0 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f69bc015940 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f69bc0157d0 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 10 client.16320 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cf51dceea0 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69bc04a030 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f69bc006080 con 0x55cf51ec1de0 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 10 client.16320 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 10 client.16320 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 20 client.16320 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ee4180 0x55cf51ee65a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 4 client.16320 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:11:01.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.682+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55cf51c57cf0 con 0x55cf51ee4180 2024-04-06T14:11:01.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.683+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ee4180 0x55cf51ee65a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.688+0000 7f69c6ffd700 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ee4180 0x55cf51ee65a0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a5737a81eb14a67b server_cookie=30fb991be8144ba3 in_seq=0 out_seq=0 2024-04-06T14:11:01.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.688+0000 7f69aaff5700 10 client.16320 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:01.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.688+0000 7f69aaff5700 1 -- 172.21.15.90:0/1211485186 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55cf51c57cf0 con 0x55cf51ee4180 2024-04-06T14:11:01.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.688+0000 7f69aaff5700 10 client.16320 handle_command_reply: tid=0 2024-04-06T14:11:01.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.688+0000 7f69d96efb80 1 client.16320 shutdown 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 msgr2=0x55cf51edb690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 secure :-1 s=READY pgs=573 cs=0 l=1 rev1=1 crypto rx=0x55cf51ec6180 tx=0x7f69bc015cd0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 shutdown_connections 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf51ed6940 0x55cf51ed6dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf51ee4180 0x55cf51ee65a0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf51eb80a0 0x55cf51edb150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 --2- 172.21.15.90:0/1211485186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf51ec1de0 0x55cf51edb690 secure :-1 s=CLOSED pgs=573 cs=0 l=1 rev1=1 crypto rx=0x55cf51ec6180 tx=0x7f69bc015cd0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.693 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.692+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 >> 172.21.15.90:0/1211485186 conn(0x55cf51e166a0 msgr2=0x55cf51ec0470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.693+0000 7f69aaff5700 0 client.16320 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:01.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.693+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 shutdown_connections 2024-04-06T14:11:01.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.694+0000 7f69d96efb80 1 -- 172.21.15.90:0/1211485186 wait complete. 2024-04-06T14:11:01.694 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.694+0000 7f69d96efb80 20 client.16320 trim_cache size 0 max 16384 2024-04-06T14:11:01.695 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:01.695 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:11:01.695 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:11:01.695 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:01.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.698+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 msgr2=0x7f69b00a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.698+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 0x7f69b00a3850 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f69c81a0ae0 tx=0x7f69bc009190 comp rx=0 tx=0).stop 2024-04-06T14:11:01.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.698+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 msgr2=0x7f69c81a57f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.698+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 secure :-1 s=READY pgs=570 cs=0 l=1 rev1=1 crypto rx=0x7f69b800e9a0 tx=0x7f69b800ecb0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 shutdown_connections 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f69b00a1390 0x7f69b00a3850 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f69c8113f00 0x7f69c819f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f69c8107db0 0x7f69c81a57f0 unknown :-1 s=CLOSED pgs=570 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 --2- 172.21.15.90:0/1856630250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f69c81086d0 0x7f69c81a5d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.699+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 >> 172.21.15.90:0/1856630250 conn(0x7f69c806be00 msgr2=0x7f69c806f9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.700+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 shutdown_connections 2024-04-06T14:11:01.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.700+0000 7f69cd53b700 1 -- 172.21.15.90:0/1856630250 wait complete. 2024-04-06T14:11:01.724 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:11:01.724 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:11:01.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.967+0000 7f41c15f0700 1 -- 172.21.15.90:0/516016590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 msgr2=0x7f41bc1139c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.973 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.967+0000 7f41c15f0700 1 --2- 172.21.15.90:0/516016590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 0x7f41bc1139c0 secure :-1 s=READY pgs=574 cs=0 l=1 rev1=1 crypto rx=0x7f41b400b0a0 tx=0x7f41b400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.977+0000 7f41c15f0700 1 -- 172.21.15.90:0/516016590 shutdown_connections 2024-04-06T14:11:01.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.977+0000 7f41c15f0700 1 --2- 172.21.15.90:0/516016590 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc113f00 0x7f41bc1162f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.977+0000 7f41c15f0700 1 --2- 172.21.15.90:0/516016590 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc107db0 0x7f41bc108190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.977+0000 7f41c15f0700 1 --2- 172.21.15.90:0/516016590 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 0x7f41bc1139c0 unknown :-1 s=CLOSED pgs=574 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.977+0000 7f41c15f0700 1 -- 172.21.15.90:0/516016590 >> 172.21.15.90:0/516016590 conn(0x7f41bc06bd30 msgr2=0x7f41bc06c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.983+0000 7f41c15f0700 1 -- 172.21.15.90:0/516016590 shutdown_connections 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.983+0000 7f41c15f0700 1 -- 172.21.15.90:0/516016590 wait complete. 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.983+0000 7f41c15f0700 1 Processor -- start 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 -- start start 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 0x7f41bc1138e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 0x7f41bc110960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 0x7f41bc110f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc111700 con 0x7f41bc1086d0 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc079010 con 0x7f41bc107db0 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.984+0000 7f41c15f0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41bc0791d0 con 0x7f41bc113f00 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.987+0000 7f41bb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 0x7f41bc110f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.988+0000 7f41bb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 0x7f41bc110f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:43938/0 (socket says 172.21.15.90:43938) 2024-04-06T14:11:01.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.988+0000 7f41bb7fe700 1 -- 172.21.15.90:0/501913579 learned_addr learned my addr 172.21.15.90:0/501913579 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 0x7f41bc1138e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 -- 172.21.15.90:0/501913579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 msgr2=0x7f41bc110f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 0x7f41bc110f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 -- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 msgr2=0x7f41bc110960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 0x7f41bc110960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 -- 172.21.15.90:0/501913579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41b4009d00 con 0x7f41bc107db0 2024-04-06T14:11:01.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.989+0000 7f41baffd700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 0x7f41bc1138e0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00b450 tx=0x7f41ac00b760 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.990+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41ac011830 con 0x7f41bc107db0 2024-04-06T14:11:01.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.990+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f41bc0794b0 con 0x7f41bc107db0 2024-04-06T14:11:01.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.990+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41bc079a10 con 0x7f41bc107db0 2024-04-06T14:11:01.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.991+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f41bc1094a0 con 0x7f41bc107db0 2024-04-06T14:11:01.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.991+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f41ac011e50 con 0x7f41bc107db0 2024-04-06T14:11:01.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.991+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f41ac010de0 con 0x7f41bc107db0 2024-04-06T14:11:01.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.994+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f41ac010480 con 0x7f41bc107db0 2024-04-06T14:11:01.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.995+0000 7f41a3fff700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 0x7f41a40a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:01.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.995+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f41ac0ccb70 con 0x7f41bc107db0 2024-04-06T14:11:01.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.997+0000 7f41ba7fc700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 0x7f41a40a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:01.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.998+0000 7f41ba7fc700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 0x7f41a40a3890 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f41b400b780 tx=0x7f41b4009450 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:01.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:01.998+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f41ac0960a0 con 0x7f41bc107db0 2024-04-06T14:11:02.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:01 smithi090 ceph-mon[43420]: pgmap v172: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-04-06T14:11:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:01 smithi134 ceph-mon[47549]: pgmap v172: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-04-06T14:11:02.254 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:01 smithi105 ceph-mon[46969]: pgmap v172: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-04-06T14:11:02.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.376+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f41bc067a20 con 0x7f41bc107db0 2024-04-06T14:11:02.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.377+0000 7f41a3fff700 1 -- 172.21.15.90:0/501913579 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f41ac015090 con 0x7f41bc107db0 2024-04-06T14:11:02.378 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:11:02.378 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:11:02.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.380+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 msgr2=0x7f41a40a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.380+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 0x7f41a40a3890 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f41b400b780 tx=0x7f41b4009450 comp rx=0 tx=0).stop 2024-04-06T14:11:02.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.380+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 msgr2=0x7f41bc1138e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.380+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 0x7f41bc1138e0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f41ac00b450 tx=0x7f41ac00b760 comp rx=0 tx=0).stop 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 shutdown_connections 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41a40a13d0 0x7f41a40a3890 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f41bc113f00 0x7f41bc110f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f41bc107db0 0x7f41bc1138e0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 --2- 172.21.15.90:0/501913579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f41bc1086d0 0x7f41bc110960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 >> 172.21.15.90:0/501913579 conn(0x7f41bc06bd30 msgr2=0x7f41bc070b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 shutdown_connections 2024-04-06T14:11:02.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.381+0000 7f41c15f0700 1 -- 172.21.15.90:0/501913579 wait complete. 2024-04-06T14:11:02.383 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:11:02.398 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:02.398 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:11:02.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.571+0000 7f9b989a8700 1 -- 172.21.15.90:0/1832815390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 msgr2=0x7f9b941162f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.571+0000 7f9b989a8700 1 --2- 172.21.15.90:0/1832815390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941162f0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f9b8c00b0a0 tx=0x7f9b8c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 -- 172.21.15.90:0/1832815390 shutdown_connections 2024-04-06T14:11:02.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 --2- 172.21.15.90:0/1832815390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941162f0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 --2- 172.21.15.90:0/1832815390 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b94107db0 0x7f9b94108190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 --2- 172.21.15.90:0/1832815390 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b941086d0 0x7f9b941139c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 -- 172.21.15.90:0/1832815390 >> 172.21.15.90:0/1832815390 conn(0x7f9b9406be00 msgr2=0x7f9b9406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 -- 172.21.15.90:0/1832815390 shutdown_connections 2024-04-06T14:11:02.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.572+0000 7f9b989a8700 1 -- 172.21.15.90:0/1832815390 wait complete. 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 Processor -- start 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 -- start start 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b94107db0 0x7f9b941a6140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941aa9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b941ab010 con 0x7f9b94107db0 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b941ab180 con 0x7f9b94113f00 2024-04-06T14:11:02.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b989a8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b941ab340 con 0x7f9b941086d0 2024-04-06T14:11:02.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b91d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b92d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941aa9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b91d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:43942/0 (socket says 172.21.15.90:43942) 2024-04-06T14:11:02.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.573+0000 7f9b91d9b700 1 -- 172.21.15.90:0/2042011384 learned_addr learned my addr 172.21.15.90:0/2042011384 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 -- 172.21.15.90:0/2042011384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 msgr2=0x7f9b941aa9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941aa9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 -- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b94107db0 msgr2=0x7f9b941a6140 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b94107db0 0x7f9b941a6140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 -- 172.21.15.90:0/2042011384 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b84009520 con 0x7f9b941086d0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9b8800dad0 tx=0x7f9b8800dde0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b837fe700 1 -- 172.21.15.90:0/2042011384 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b8800ca90 con 0x7f9b941086d0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b8c009d00 con 0x7f9b941086d0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.574+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b941ab810 con 0x7f9b941086d0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.575+0000 7f9b837fe700 1 -- 172.21.15.90:0/2042011384 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b8800cbf0 con 0x7f9b941086d0 2024-04-06T14:11:02.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.575+0000 7f9b837fe700 1 -- 172.21.15.90:0/2042011384 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b88014610 con 0x7f9b941086d0 2024-04-06T14:11:02.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.579+0000 7f9b837fe700 1 -- 172.21.15.90:0/2042011384 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9b88014770 con 0x7f9b941086d0 2024-04-06T14:11:02.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.579+0000 7f9ba4b5cb80 1 Processor -- start 2024-04-06T14:11:02.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.579+0000 7f9ba4b5cb80 1 -- start start 2024-04-06T14:11:02.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5563b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b55701d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b54bb990 con 0x5631b5563780 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b54b9350 con 0x5631b55640a0 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b54b4c40 con 0x5631b556dde0 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43792/0 (socket says 172.21.15.90:43792) 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9b9259c700 1 -- 172.21.15.90:0/716339353 learned_addr learned my addr 172.21.15.90:0/716339353 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b55701d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.580+0000 7f9b837fe700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 0x7f9b7c0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 -- 172.21.15.90:0/716339353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 msgr2=0x5631b55701d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b55701d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 -- 172.21.15.90:0/716339353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 msgr2=0x5631b5563b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5563b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 0x7f9b7c0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 -- 172.21.15.90:0/716339353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5631b53363b0 con 0x5631b55640a0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b837fe700 1 -- 172.21.15.90:0/2042011384 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9b880cc500 con 0x7f9b941086d0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b9259c700 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9b84009d30 tx=0x7f9b8400e630 comp rx=0 tx=0).ready entity=mon.2 client_cookie=447e72628116c4d4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 0x7f9b7c0a3890 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f9b8c0060b0 tx=0x7f9b8c006000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b80ff9700 1 -- 172.21.15.90:0/716339353 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b8400ee40 con 0x5631b55640a0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b80ff9700 1 -- 172.21.15.90:0/716339353 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b84009e80 con 0x5631b55640a0 2024-04-06T14:11:02.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.581+0000 7f9b80ff9700 1 -- 172.21.15.90:0/716339353 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b84010430 con 0x5631b55640a0 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/716339353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 msgr2=0x5631b556d8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f9b84009d30 tx=0x7f9b8400e630 comp rx=0 tx=0).stop 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/716339353 shutdown_connections 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b55701d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5563b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/716339353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d8a0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.582+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/716339353 >> 172.21.15.90:0/716339353 conn(0x5631b54c26a0 msgr2=0x5631b555ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/716339353 shutdown_connections 2024-04-06T14:11:02.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/716339353 wait complete. 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- start start 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 Processor -- start 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 0x5631b556d7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 0x5631b556abb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b5450d80 con 0x5631b55640a0 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b5450a20 con 0x5631b5563780 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b53386a0 con 0x5631b556dde0 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.584+0000 7f9ba4b5cb80 10 client.0 _ll_register_callbacks cb 0x5631b545a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 0x5631b556d7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 0x5631b556d7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43800/0 (socket says 172.21.15.90:43800) 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:43964/0 (socket says 172.21.15.90:43964) 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b9259c700 1 -- 172.21.15.90:0/147088509 learned_addr learned my addr 172.21.15.90:0/147088509 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 0x5631b556abb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 -- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 msgr2=0x5631b556d7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 0x5631b556d7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 -- 172.21.15.90:0/147088509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 msgr2=0x5631b556abb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 0x5631b556abb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 -- 172.21.15.90:0/147088509 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b840099e0 con 0x5631b556dde0 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 0x5631b556abb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f9b8c009fd0 tx=0x7f9b8c00b3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8be33a01b65fe3f7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:02.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b8c00e040 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.585+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9b8c0076f0 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.586+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5631b54ad700 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.586+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b8c003e60 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.586+0000 7f9ba4b5cb80 10 client.25951 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.586+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5631b52e2b60 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.586+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5631b54ad700 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b8c028470 con 0x5631b556dde0 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 10 client.25951 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 10 client.25951 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 20 client.25951 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:02.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b558e9c0 0x5631b5590de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 4 client.25951 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5631b5303cf0 con 0x5631b558e9c0 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.587+0000 7f9b9259c700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b558e9c0 0x5631b5590de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9b79ffb700 10 client.25951 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9b9259c700 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b558e9c0 0x5631b5590de0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=eb8d4782627236fa server_cookie=e922c7d330527d44 in_seq=0 out_seq=0 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9b79ffb700 1 -- 172.21.15.90:0/147088509 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5631b5303cf0 con 0x5631b558e9c0 2024-04-06T14:11:02.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9b79ffb700 10 client.25951 handle_command_reply: tid=0 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 client.25951 shutdown 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 msgr2=0x5631b556b1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f9b8c009fd0 tx=0x7f9b8c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 shutdown_connections 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b558e9c0 0x5631b5590de0 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b556b1c0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b5563780 0x5631b556d7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/147088509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b55640a0 0x5631b556abb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.588+0000 7f9b79ffb700 0 client.25951 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.589+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 >> 172.21.15.90:0/147088509 conn(0x5631b54c26a0 msgr2=0x5631b555dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.590+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 shutdown_connections 2024-04-06T14:11:02.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.590+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/147088509 wait complete. 2024-04-06T14:11:02.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.590+0000 7f9ba4b5cb80 20 client.25951 trim_cache size 0 max 16384 2024-04-06T14:11:02.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.615+0000 7f9ba4b5cb80 1 Processor -- start 2024-04-06T14:11:02.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.615+0000 7f9ba4b5cb80 1 -- start start 2024-04-06T14:11:02.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5574ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5565e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b555f700 con 0x5631b5563780 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b5575000 con 0x5631b55640a0 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b55690e0 con 0x5631b556dde0 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43804/0 (socket says 172.21.15.90:43804) 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 -- 172.21.15.90:0/3603646844 learned_addr learned my addr 172.21.15.90:0/3603646844 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 -- 172.21.15.90:0/3603646844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 msgr2=0x5631b5565e40 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5565e40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 -- 172.21.15.90:0/3603646844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 msgr2=0x5631b5574ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5574ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.616+0000 7f9b91d9b700 1 -- 172.21.15.90:0/3603646844 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5631b547aea0 con 0x5631b55640a0 2024-04-06T14:11:02.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.617+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9b88099570 tx=0x7f9b880b9040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=450744dfac599482 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.617+0000 7f9b817fa700 1 -- 172.21.15.90:0/3603646844 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b88099ce0 con 0x5631b55640a0 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.617+0000 7f9b817fa700 1 -- 172.21.15.90:0/3603646844 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b880995e0 con 0x5631b55640a0 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.617+0000 7f9b817fa700 1 -- 172.21.15.90:0/3603646844 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b88097980 con 0x5631b55640a0 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/3603646844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 msgr2=0x5631b556d350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f9b88099570 tx=0x7f9b880b9040 comp rx=0 tx=0).stop 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/3603646844 shutdown_connections 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5565e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5574ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/3603646844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b556d350 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/3603646844 >> 172.21.15.90:0/3603646844 conn(0x5631b54c26a0 msgr2=0x5631b555a890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/3603646844 shutdown_connections 2024-04-06T14:11:02.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.618+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/3603646844 wait complete. 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 -- start start 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 Processor -- start 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5573380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b55738c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5573e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b55e29b0 con 0x5631b5563780 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b55e2b20 con 0x5631b55640a0 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5631b55e2ce0 con 0x5631b556dde0 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9ba4b5cb80 10 client.0 _ll_register_callbacks cb 0x5631b545a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5573380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9b9259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5573380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47892/0 (socket says 172.21.15.90:47892) 2024-04-06T14:11:02.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9b9259c700 1 -- 172.21.15.90:0/127260163 learned_addr learned my addr 172.21.15.90:0/127260163 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9b92d9d700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5573e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.619+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b55738c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 -- 172.21.15.90:0/127260163 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 msgr2=0x5631b5573e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5573e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 -- 172.21.15.90:0/127260163 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 msgr2=0x5631b5573380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5573380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 -- 172.21.15.90:0/127260163 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b941ab810 con 0x5631b55640a0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b91d9b700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b55738c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f9b88099570 tx=0x7f9b880c9bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc362be340213a93 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b880842b0 con 0x5631b55640a0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9b880b3070 con 0x5631b55640a0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5631b5475380 con 0x5631b55640a0 2024-04-06T14:11:02.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b88099ce0 con 0x5631b55640a0 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9ba4b5cb80 10 client.26054 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.620+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5631b533acd0 con 0x5631b55640a0 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b8807c080 con 0x5631b55640a0 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b880b7bd0 con 0x5631b55640a0 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 10 client.26054 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 10 client.26054 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 20 client.26054 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b55e4200 0x5631b55e65f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 4 client.26054 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:02.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.621+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5631b5338ce0 con 0x5631b55e4200 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.622+0000 7f9b9259c700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b55e4200 0x5631b55e65f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9b79ffb700 10 client.26054 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9b9259c700 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b55e4200 0x5631b55e65f0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d60d371ba3086c80 server_cookie=78acfcf06771f30c in_seq=0 out_seq=0 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9b79ffb700 1 -- 172.21.15.90:0/127260163 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5631b5338ce0 con 0x5631b55e4200 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9b79ffb700 10 client.26054 handle_command_reply: tid=0 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 client.26054 shutdown 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 msgr2=0x5631b55738c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b55738c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f9b88099570 tx=0x7f9b880c9bc0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 shutdown_connections 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5631b55e4200 0x5631b55e65f0 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5631b556dde0 0x5631b5573e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9b79ffb700 0 client.26054 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5631b5563780 0x5631b5573380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 --2- 172.21.15.90:0/127260163 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5631b55640a0 0x5631b55738c0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 >> 172.21.15.90:0/127260163 conn(0x5631b54c26a0 msgr2=0x5631b5565e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 shutdown_connections 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 1 -- 172.21.15.90:0/127260163 wait complete. 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.623+0000 7f9ba4b5cb80 20 client.26054 trim_cache size 0 max 16384 2024-04-06T14:11:02.627 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:02.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.626+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 msgr2=0x7f9b7c0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.626+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 0x7f9b7c0a3890 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f9b8c0060b0 tx=0x7f9b8c006000 comp rx=0 tx=0).stop 2024-04-06T14:11:02.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.626+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 msgr2=0x7f9b941a6680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.626+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f9b8800dad0 tx=0x7f9b8800dde0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.631+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 shutdown_connections 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.631+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b7c0a13d0 0x7f9b7c0a3890 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.631+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b94113f00 0x7f9b941aa9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.631+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b94107db0 0x7f9b941a6140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.631+0000 7f9b989a8700 1 --2- 172.21.15.90:0/2042011384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b941086d0 0x7f9b941a6680 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.632+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 >> 172.21.15.90:0/2042011384 conn(0x7f9b9406be00 msgr2=0x7f9b9406fa50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.632+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 shutdown_connections 2024-04-06T14:11:02.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.632+0000 7f9b989a8700 1 -- 172.21.15.90:0/2042011384 wait complete. 2024-04-06T14:11:02.647 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:02.647 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:11:02.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4094026297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6481089d0 msgr2=0x7ff64810f1f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4094026297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6481089d0 0x7ff64810f1f0 secure :-1 s=READY pgs=575 cs=0 l=1 rev1=1 crypto rx=0x7ff6380099c0 tx=0x7ff638009cd0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4094026297 shutdown_connections 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4094026297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff64810f920 0x7ff6481162a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4094026297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff648108020 0x7ff648108400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4094026297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6481089d0 0x7ff64810f1f0 unknown :-1 s=CLOSED pgs=575 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4094026297 >> 172.21.15.90:0/4094026297 conn(0x7ff64806bc20 msgr2=0x7ff64806c030 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4094026297 shutdown_connections 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4094026297 wait complete. 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.839+0000 7ff64cf9d700 1 Processor -- start 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 -- start start 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 0x7ff64819f7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 0x7ff6481a34b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6481a39f0 con 0x7ff64810f920 2024-04-06T14:11:02.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6481a3b60 con 0x7ff648108020 2024-04-06T14:11:02.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff64cf9d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6481a3d20 con 0x7ff6481a3000 2024-04-06T14:11:02.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff646ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff6477fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 0x7ff64819f7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff646ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47906/0 (socket says 172.21.15.90:47906) 2024-04-06T14:11:02.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff6477fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 0x7ff64819f7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43822/0 (socket says 172.21.15.90:43822) 2024-04-06T14:11:02.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff646ffd700 1 -- 172.21.15.90:0/4193622893 learned_addr learned my addr 172.21.15.90:0/4193622893 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.840+0000 7ff647fff700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 0x7ff6481a34b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.841+0000 7ff646ffd700 1 -- 172.21.15.90:0/4193622893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 msgr2=0x7ff6481a34b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.841+0000 7ff646ffd700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 0x7ff6481a34b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.841+0000 7ff646ffd700 1 -- 172.21.15.90:0/4193622893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 msgr2=0x7ff64819f7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.841+0000 7ff646ffd700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 0x7ff64819f7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.841+0000 7ff646ffd700 1 -- 172.21.15.90:0/4193622893 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6380095f0 con 0x7ff64810f920 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.842+0000 7ff647fff700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 0x7ff6481a34b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.842+0000 7ff646ffd700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7ff638000c00 tx=0x7ff63800f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.842+0000 7ff644ff9700 1 -- 172.21.15.90:0/4193622893 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff63801d070 con 0x7ff64810f920 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.842+0000 7ff644ff9700 1 -- 172.21.15.90:0/4193622893 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff638022470 con 0x7ff64810f920 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.843+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff6481ac390 con 0x7ff64810f920 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.843+0000 7ff644ff9700 1 -- 172.21.15.90:0/4193622893 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff638017650 con 0x7ff64810f920 2024-04-06T14:11:02.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.843+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6481ac7b0 con 0x7ff64810f920 2024-04-06T14:11:02.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.846+0000 7ff644ff9700 1 -- 172.21.15.90:0/4193622893 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff638022850 con 0x7ff64810f920 2024-04-06T14:11:02.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.846+0000 7ff644ff9700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 0x7ff6300a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.847+0000 7ff6477fe700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 0x7ff6300a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.847+0000 7ff644ff9700 1 -- 172.21.15.90:0/4193622893 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff6380cd770 con 0x7ff64810f920 2024-04-06T14:11:02.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.847+0000 7ff6477fe700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 0x7ff6300a3800 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7ff640009380 tx=0x7ff64000b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 Processor -- start 2024-04-06T14:11:02.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 -- start start 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 0x55c9d6832bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d6833100 0x55c9d683da70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d68403a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d678b990 con 0x55c9d68327e0 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d6789350 con 0x55c9d6833100 2024-04-06T14:11:02.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d6784c40 con 0x55c9d683dfb0 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff6477fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 0x55c9d6832bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff6477fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 0x55c9d6832bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47922/0 (socket says 172.21.15.90:47922) 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff6477fe700 1 -- 172.21.15.90:0/887135989 learned_addr learned my addr 172.21.15.90:0/887135989 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff647fff700 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d68403a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.848+0000 7ff646ffd700 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d6833100 0x55c9d683da70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 -- 172.21.15.90:0/887135989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d6833100 msgr2=0x55c9d683da70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d6833100 0x55c9d683da70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 -- 172.21.15.90:0/887135989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 msgr2=0x55c9d6832bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 0x55c9d6832bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 -- 172.21.15.90:0/887135989 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c9d66063b0 con 0x55c9d683dfb0 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff647fff700 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d68403a0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff63c005950 tx=0x7ff63c0106b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bb55be8f874f5154 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.849+0000 7ff62dffb700 1 -- 172.21.15.90:0/887135989 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff63c004250 con 0x55c9d683dfb0 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.850+0000 7ff62dffb700 1 -- 172.21.15.90:0/887135989 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff63c010ea0 con 0x55c9d683dfb0 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.850+0000 7ff62dffb700 1 -- 172.21.15.90:0/887135989 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff63c0054e0 con 0x55c9d683dfb0 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/887135989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 msgr2=0x55c9d68403a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d68403a0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7ff63c005950 tx=0x7ff63c0106b0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/887135989 shutdown_connections 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d68403a0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d68327e0 0x55c9d6832bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/887135989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d6833100 0x55c9d683da70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.851+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/887135989 >> 172.21.15.90:0/887135989 conn(0x55c9d67926a0 msgr2=0x55c9d682ec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.872+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/887135989 shutdown_connections 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.876+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/887135989 wait complete. 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.877+0000 7ff65a3b3b80 1 -- start start 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.877+0000 7ff65a3b3b80 1 Processor -- start 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d683ad70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683dfb0 0x55c9d683b2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 0x55c9d6849ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d6720d80 con 0x55c9d683dfb0 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d6720a20 con 0x55c9d68327e0 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d66086a0 con 0x55c9d683b7f0 2024-04-06T14:11:02.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.878+0000 7ff65a3b3b80 10 client.0 _ll_register_callbacks cb 0x55c9d672a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:02.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.881+0000 7ff6477fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d683ad70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.881+0000 7ff647fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 0x55c9d6849ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.888+0000 7ff647fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 0x55c9d6849ce0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44012/0 (socket says 172.21.15.90:44012) 2024-04-06T14:11:02.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.888+0000 7ff647fff700 1 -- 172.21.15.90:0/82623057 learned_addr learned my addr 172.21.15.90:0/82623057 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.888+0000 7ff646ffd700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683dfb0 0x55c9d683b2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 -- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 msgr2=0x55c9d6849ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 0x55c9d6849ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 -- 172.21.15.90:0/82623057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683dfb0 msgr2=0x55c9d683b2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683dfb0 0x55c9d683b2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 -- 172.21.15.90:0/82623057 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff63c009c70 con 0x55c9d68327e0 2024-04-06T14:11:02.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.889+0000 7ff6477fe700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d683ad70 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7ff640007b70 tx=0x7ff640004ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6ce9dc37a74bc681 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.892+0000 7ff626ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:02.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.898+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff64000e050 con 0x55c9d68327e0 2024-04-06T14:11:02.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.898+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff640016e10 con 0x55c9d68327e0 2024-04-06T14:11:02.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.898+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c9d6745380 con 0x55c9d68327e0 2024-04-06T14:11:02.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.899+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff640022420 con 0x55c9d68327e0 2024-04-06T14:11:02.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.901+0000 7ff65a3b3b80 10 client.26066 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:02.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.901+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c9d65b2b60 con 0x55c9d68327e0 2024-04-06T14:11:02.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c9d6745380 con 0x55c9d68327e0 2024-04-06T14:11:02.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff64002a650 con 0x55c9d68327e0 2024-04-06T14:11:02.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 10 client.26066 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:02.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 10 client.26066 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:11:02.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 20 client.26066 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:02.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d6870020 0x55c9d6872440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 4 client.26066 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:02.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.905+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c9d677d700 con 0x55c9d6870020 2024-04-06T14:11:02.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.906+0000 7ff646ffd700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d6870020 0x55c9d6872440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.910+0000 7ff646ffd700 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d6870020 0x55c9d6872440 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=263676166a2c5080 server_cookie=9370b1fdc9aad034 in_seq=0 out_seq=0 2024-04-06T14:11:02.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.910+0000 7ff626ffd700 10 client.26066 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:02.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.911+0000 7ff626ffd700 1 -- 172.21.15.90:0/82623057 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c9d677d700 con 0x55c9d6870020 2024-04-06T14:11:02.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.911+0000 7ff626ffd700 10 client.26066 handle_command_reply: tid=0 2024-04-06T14:11:02.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.911+0000 7ff65a3b3b80 1 client.26066 shutdown 2024-04-06T14:11:02.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.914+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 msgr2=0x55c9d683ad70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.914+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d683ad70 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7ff640007b70 tx=0x7ff640004ad0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 shutdown_connections 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d6870020 0x55c9d6872440 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683b7f0 0x55c9d6849ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d683ad70 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/82623057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683dfb0 0x55c9d683b2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.920+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 >> 172.21.15.90:0/82623057 conn(0x55c9d67926a0 msgr2=0x55c9d682e820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.922+0000 7ff626ffd700 0 client.26066 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:02.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.927+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 shutdown_connections 2024-04-06T14:11:02.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.931+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/82623057 wait complete. 2024-04-06T14:11:02.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.932+0000 7ff65a3b3b80 20 client.26066 trim_cache size 0 max 16384 2024-04-06T14:11:02.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.951+0000 7ff65a3b3b80 1 Processor -- start 2024-04-06T14:11:02.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.951+0000 7ff65a3b3b80 1 -- start start 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.951+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d6844c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.951+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d683ad30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d683b270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d683db20 con 0x55c9d683b7f0 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d68346a0 con 0x55c9d68327e0 2024-04-06T14:11:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d683bd10 con 0x55c9d683dfb0 2024-04-06T14:11:02.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff647fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d683b270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff647fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d683b270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44022/0 (socket says 172.21.15.90:44022) 2024-04-06T14:11:02.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.952+0000 7ff647fff700 1 -- 172.21.15.90:0/3827576949 learned_addr learned my addr 172.21.15.90:0/3827576949 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d6844c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff646ffd700 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d683ad30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 -- 172.21.15.90:0/3827576949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 msgr2=0x55c9d683b270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d683b270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 -- 172.21.15.90:0/3827576949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 msgr2=0x55c9d683ad30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d683ad30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 -- 172.21.15.90:0/3827576949 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c9d674aea0 con 0x55c9d68327e0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff6477fe700 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d6844c30 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7ff64002d040 tx=0x7ff640009e60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=15726e9ac1ada175 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff62e7fc700 1 -- 172.21.15.90:0/3827576949 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff640038070 con 0x55c9d68327e0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff62e7fc700 1 -- 172.21.15.90:0/3827576949 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff64000f040 con 0x55c9d68327e0 2024-04-06T14:11:02.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.953+0000 7ff62e7fc700 1 -- 172.21.15.90:0/3827576949 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff64000e040 con 0x55c9d68327e0 2024-04-06T14:11:02.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/3827576949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 msgr2=0x55c9d6844c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d6844c30 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7ff64002d040 tx=0x7ff640009e60 comp rx=0 tx=0).stop 2024-04-06T14:11:02.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/3827576949 shutdown_connections 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d683dfb0 0x55c9d683b270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d68327e0 0x55c9d6844c30 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/3827576949 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d683ad30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/3827576949 >> 172.21.15.90:0/3827576949 conn(0x55c9d67926a0 msgr2=0x55c9d68360b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/3827576949 shutdown_connections 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.954+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/3827576949 wait complete. 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.955+0000 7ff65a3b3b80 1 -- start start 2024-04-06T14:11:02.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.955+0000 7ff65a3b3b80 1 Processor -- start 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d68327e0 0x55c9d6842e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d6843340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d6843a20 con 0x55c9d683b7f0 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d68c3390 con 0x55c9d683dfb0 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9d68c34d0 con 0x55c9d68327e0 2024-04-06T14:11:02.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff65a3b3b80 10 client.0 _ll_register_callbacks cb 0x55c9d672a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43850/0 (socket says 172.21.15.90:43850) 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 -- 172.21.15.90:0/1044085091 learned_addr learned my addr 172.21.15.90:0/1044085091 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff6477fe700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d68327e0 0x55c9d6842e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 -- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d68327e0 msgr2=0x55c9d6842e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d68327e0 0x55c9d6842e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff646ffd700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d6843340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 -- 172.21.15.90:0/1044085091 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 msgr2=0x55c9d6843340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d6843340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff647fff700 1 -- 172.21.15.90:0/1044085091 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c9d65b2b60 con 0x55c9d683dfb0 2024-04-06T14:11:02.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.958+0000 7ff646ffd700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d6843340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff647fff700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7ff63c0097a0 tx=0x7ff63c0038b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b66e6cce3d9be783 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff63c01e040 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7ff63c023460 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff63c010ea0 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff63c023620 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 10 client.26072 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c9d660acd0 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff63c010ea0 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff63c0117a0 con 0x55c9d683dfb0 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 10 client.26072 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 10 client.26072 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:11:02.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 20 client.26072 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:02.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d68c4b70 0x55c9d68c6f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:02.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 4 client.26072 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:02.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.959+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c9d65d3cf0 con 0x55c9d68c4b70 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.960+0000 7ff6477fe700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d68c4b70 0x55c9d68c6f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff626ffd700 10 client.26072 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff6477fe700 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d68c4b70 0x55c9d68c6f90 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=8a15f87ba8835f62 server_cookie=709a7cfc1a9ed097 in_seq=0 out_seq=0 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff626ffd700 1 -- 172.21.15.90:0/1044085091 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c9d65d3cf0 con 0x55c9d68c4b70 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff626ffd700 10 client.26072 handle_command_reply: tid=0 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 client.26072 shutdown 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 msgr2=0x55c9d68c2e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7ff63c0097a0 tx=0x7ff63c0038b0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 shutdown_connections 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c9d68c4b70 0x55c9d68c6f90 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c9d683dfb0 0x55c9d68c2e50 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c9d68327e0 0x55c9d6842e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 --2- 172.21.15.90:0/1044085091 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c9d683b7f0 0x55c9d6843340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.961+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 >> 172.21.15.90:0/1044085091 conn(0x55c9d67926a0 msgr2=0x55c9d683aca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff626ffd700 0 client.26072 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 shutdown_connections 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff65a3b3b80 1 -- 172.21.15.90:0/1044085091 wait complete. 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff65a3b3b80 20 client.26072 trim_cache size 0 max 16384 2024-04-06T14:11:02.965 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:02.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 msgr2=0x7ff6300a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.966 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 0x7ff6300a3800 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7ff640009380 tx=0x7ff64000b040 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 msgr2=0x7ff64819fd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 secure :-1 s=READY pgs=576 cs=0 l=1 rev1=1 crypto rx=0x7ff638000c00 tx=0x7ff63800f690 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 shutdown_connections 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6300a1340 0x7ff6300a3800 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6481a3000 0x7ff6481a34b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff648108020 0x7ff64819f7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 --2- 172.21.15.90:0/4193622893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff64810f920 0x7ff64819fd10 unknown :-1 s=CLOSED pgs=576 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.962+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 >> 172.21.15.90:0/4193622893 conn(0x7ff64806bc20 msgr2=0x7ff64810d500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.963+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 shutdown_connections 2024-04-06T14:11:02.967 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:02.963+0000 7ff64cf9d700 1 -- 172.21.15.90:0/4193622893 wait complete. 2024-04-06T14:11:02.976 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:02 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/501913579' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:02.977 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:02.977 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:11:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 315.0 s, 1564.2 tps, lat 64.123 ms stddev 159.083 2024-04-06T14:11:03.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.140+0000 7f9efaf84700 1 -- 172.21.15.90:0/2450370128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef40ffe70 msgr2=0x7f9ef4100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.140+0000 7f9efaf84700 1 --2- 172.21.15.90:0/2450370128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef40ffe70 0x7f9ef4100250 secure :-1 s=READY pgs=577 cs=0 l=1 rev1=1 crypto rx=0x7f9ee8009990 tx=0x7f9ee8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 -- 172.21.15.90:0/2450370128 shutdown_connections 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 --2- 172.21.15.90:0/2450370128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef4104f80 0x7f9ef410cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 --2- 172.21.15.90:0/2450370128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef40ffe70 0x7f9ef4100250 unknown :-1 s=CLOSED pgs=577 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 --2- 172.21.15.90:0/2450370128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4100820 0x7f9ef4104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 -- 172.21.15.90:0/2450370128 >> 172.21.15.90:0/2450370128 conn(0x7f9ef4076d40 msgr2=0x7f9ef4077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 -- 172.21.15.90:0/2450370128 shutdown_connections 2024-04-06T14:11:03.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.141+0000 7f9efaf84700 1 -- 172.21.15.90:0/2450370128 wait complete. 2024-04-06T14:11:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:02 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/501913579' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 Processor -- start 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 -- start start 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 0x7f9ef406a8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 0x7f9ef406b2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ef41ac030 con 0x7f9ef406ae00 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ef41ac1a0 con 0x7f9ef4104f80 2024-04-06T14:11:03.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9efaf84700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ef41ac360 con 0x7f9ef40ffe70 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 0x7f9ef406a8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44052/0 (socket says 172.21.15.90:44052) 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.142+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 0x7f9ef406a8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43868/0 (socket says 172.21.15.90:43868) 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 -- 172.21.15.90:0/1140083780 learned_addr learned my addr 172.21.15.90:0/1140083780 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 0x7f9ef406b2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 -- 172.21.15.90:0/1140083780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 msgr2=0x7f9ef406a8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 0x7f9ef406a8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 -- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 msgr2=0x7f9ef406b2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 0x7f9ef406b2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 -- 172.21.15.90:0/1140083780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ee80095f0 con 0x7f9ef40ffe70 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 0x7f9ef406b2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f9ee8004760 tx=0x7f9ee8004790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef1ffb700 1 -- 172.21.15.90:0/1140083780 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee801d070 con 0x7f9ef40ffe70 2024-04-06T14:11:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ef4101820 con 0x7f9ef40ffe70 2024-04-06T14:11:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef1ffb700 1 -- 172.21.15.90:0/1140083780 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ee8022470 con 0x7f9ef40ffe70 2024-04-06T14:11:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.143+0000 7f9ef1ffb700 1 -- 172.21.15.90:0/1140083780 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee8017650 con 0x7f9ef40ffe70 2024-04-06T14:11:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.144+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ef4101cd0 con 0x7f9ef40ffe70 2024-04-06T14:11:03.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.146+0000 7f9ef1ffb700 1 -- 172.21.15.90:0/1140083780 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9ee8017870 con 0x7f9ef40ffe70 2024-04-06T14:11:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.146+0000 7f9ef1ffb700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 0x7f9ee40a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.147+0000 7f9ef1ffb700 1 -- 172.21.15.90:0/1140083780 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ee80cdfc0 con 0x7f9ef40ffe70 2024-04-06T14:11:03.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.147+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 0x7f9ee40a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 Processor -- start 2024-04-06T14:11:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 -- start start 2024-04-06T14:11:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23feea0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23feea5f0 0x55a23feec9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fe2f990 con 0x55a23feea5f0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fe2d350 con 0x55a23fee08b0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.148+0000 7f9f07138b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fe28c40 con 0x55a23fedff90 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 0x7f9ee40a3750 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f9ef406bff0 tx=0x7f9ee0036040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef9521700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23feea5f0 0x55a23feec9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44054/0 (socket says 172.21.15.90:44054) 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 -- 172.21.15.90:0/3612548998 learned_addr learned my addr 172.21.15.90:0/3612548998 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23feea0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 -- 172.21.15.90:0/3612548998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 msgr2=0x55a23feea0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23feea0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 -- 172.21.15.90:0/3612548998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23feea5f0 msgr2=0x55a23feec9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23feea5f0 0x55a23feec9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 -- 172.21.15.90:0/3612548998 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a23fcaa3b0 con 0x55a23fedff90 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f9ee8024000 tx=0x7f9ee809ff90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=177fc7b9ae244386 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.149+0000 7f9edeffd700 1 -- 172.21.15.90:0/3612548998 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee809acb0 con 0x55a23fedff90 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9edeffd700 1 -- 172.21.15.90:0/3612548998 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ee809aad0 con 0x55a23fedff90 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9edeffd700 1 -- 172.21.15.90:0/3612548998 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee8097880 con 0x55a23fedff90 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 -- 172.21.15.90:0/3612548998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 msgr2=0x55a23fee0370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f9ee8024000 tx=0x7f9ee809ff90 comp rx=0 tx=0).stop 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 -- 172.21.15.90:0/3612548998 shutdown_connections 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23feea5f0 0x55a23feec9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23fedff90 0x55a23fee0370 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 --2- 172.21.15.90:0/3612548998 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23feea0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 -- 172.21.15.90:0/3612548998 >> 172.21.15.90:0/3612548998 conn(0x55a23fe366a0 msgr2=0x55a23fed2a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 -- 172.21.15.90:0/3612548998 shutdown_connections 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.150+0000 7f9f07138b80 1 -- 172.21.15.90:0/3612548998 wait complete. 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 -- start start 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 Processor -- start 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 0x55a23fee73c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fee7e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fdc4d80 con 0x55a23fedff90 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fdc4a20 con 0x55a23fee08b0 2024-04-06T14:11:03.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fcac6a0 con 0x55a23feea5f0 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9f07138b80 10 client.0 _ll_register_callbacks cb 0x55a23fdce470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 0x55a23fee73c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43892/0 (socket says 172.21.15.90:43892) 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef8d20700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 0x55a23fee73c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47982/0 (socket says 172.21.15.90:47982) 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef3fff700 1 -- 172.21.15.90:0/2483101487 learned_addr learned my addr 172.21.15.90:0/2483101487 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.151+0000 7f9ef9521700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fee7e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 -- 172.21.15.90:0/2483101487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 msgr2=0x55a23fee7e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fee7e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 -- 172.21.15.90:0/2483101487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 msgr2=0x55a23fee73c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 0x55a23fee73c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 -- 172.21.15.90:0/2483101487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ef4101cd0 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f9ee0009750 tx=0x7f9ee0004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d5da967bac345fb2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ed3fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee0034220 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a23fde9380 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ee0003a90 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ee0003c50 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9f07138b80 10 client.26084 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a23fc56b60 con 0x55a23fee08b0 2024-04-06T14:11:03.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.152+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55a23fde9380 con 0x55a23fee08b0 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ee0052070 con 0x55a23fee08b0 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 10 client.26084 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 10 client.26084 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 20 client.26084 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23fef9f20 0x55a23fefc340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 4 client.26084 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a23fe21700 con 0x55a23fef9f20 2024-04-06T14:11:03.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23fef9f20 0x55a23fefc340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9ed3fff700 10 client.26084 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:03.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.153+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23fef9f20 0x55a23fefc340 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cb7ee543c203313c server_cookie=c8e2423ed950801c in_seq=0 out_seq=0 2024-04-06T14:11:03.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.154+0000 7f9ed3fff700 1 -- 172.21.15.90:0/2483101487 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a23fe21700 con 0x55a23fef9f20 2024-04-06T14:11:03.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.154+0000 7f9ed3fff700 10 client.26084 handle_command_reply: tid=0 2024-04-06T14:11:03.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.154+0000 7f9f07138b80 1 client.26084 shutdown 2024-04-06T14:11:03.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.154+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 msgr2=0x55a23fee7900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.154+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f9ee0009750 tx=0x7f9ee0004750 comp rx=0 tx=0).stop 2024-04-06T14:11:03.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 shutdown_connections 2024-04-06T14:11:03.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23fef9f20 0x55a23fefc340 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fee7e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fedff90 0x55a23fee73c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 --2- 172.21.15.90:0/2483101487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fee08b0 0x55a23fee7900 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 >> 172.21.15.90:0/2483101487 conn(0x55a23fe366a0 msgr2=0x55a23fed1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9ed3fff700 0 client.26084 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 shutdown_connections 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 1 -- 172.21.15.90:0/2483101487 wait complete. 2024-04-06T14:11:03.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.156+0000 7f9f07138b80 20 client.26084 trim_cache size 0 max 16384 2024-04-06T14:11:03.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.171+0000 7f9f07138b80 1 Processor -- start 2024-04-06T14:11:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 -- start start 2024-04-06T14:11:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23fee1e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fed23a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23feea370 con 0x55a23fee08b0 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fee5880 con 0x55a23fedff90 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9f07138b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fee59c0 con 0x55a23feea5f0 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47994/0 (socket says 172.21.15.90:47994) 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef3fff700 1 -- 172.21.15.90:0/1800372684 learned_addr learned my addr 172.21.15.90:0/1800372684 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fed23a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23fee1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 -- 172.21.15.90:0/1800372684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 msgr2=0x55a23fee1e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23fee1e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 -- 172.21.15.90:0/1800372684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 msgr2=0x55a23fee7c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef9521700 1 -- 172.21.15.90:0/1800372684 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a23fdeeea0 con 0x55a23feea5f0 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.173+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fed23a0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9eec009e10 tx=0x7f9eec009e40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=85a139c3a12c11a0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9edf7fe700 1 -- 172.21.15.90:0/1800372684 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eec007e40 con 0x55a23feea5f0 2024-04-06T14:11:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9edf7fe700 1 -- 172.21.15.90:0/1800372684 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9eec00f040 con 0x55a23feea5f0 2024-04-06T14:11:03.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9edf7fe700 1 -- 172.21.15.90:0/1800372684 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9eec039420 con 0x55a23feea5f0 2024-04-06T14:11:03.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 -- 172.21.15.90:0/1800372684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 msgr2=0x55a23fed23a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fed23a0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f9eec009e10 tx=0x7f9eec009e40 comp rx=0 tx=0).stop 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 -- 172.21.15.90:0/1800372684 shutdown_connections 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23fed23a0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23fee1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1800372684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23fee7c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.174+0000 7f9f07138b80 1 -- 172.21.15.90:0/1800372684 >> 172.21.15.90:0/1800372684 conn(0x55a23fe366a0 msgr2=0x55a23fee26f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- 172.21.15.90:0/1800372684 shutdown_connections 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- 172.21.15.90:0/1800372684 wait complete. 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- start start 2024-04-06T14:11:03.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 Processor -- start 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23feee980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23feef440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fedb9d0 con 0x55a23fee08b0 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fedbb40 con 0x55a23fedff90 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23fedbd00 con 0x55a23feea5f0 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9f07138b80 10 client.0 _ll_register_callbacks cb 0x55a23fdce470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9ef3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:48010/0 (socket says 172.21.15.90:48010) 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.175+0000 7f9ef3fff700 1 -- 172.21.15.90:0/1354797850 learned_addr learned my addr 172.21.15.90:0/1354797850 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:03.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef9521700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23feef440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 -- 172.21.15.90:0/1354797850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 msgr2=0x55a23feef440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23feef440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 -- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 msgr2=0x55a23feee980 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23feee980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 -- 172.21.15.90:0/1354797850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55a23fc56b60 con 0x55a23fee08b0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ef3fff700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f9ee00347c0 tx=0x7f9ee0054640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd8f3bcbd70420b9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ed3fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ee0003ca0 con 0x55a23fee08b0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f9ee0009260 con 0x55a23fee08b0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9ee809aad0 con 0x55a23fee08b0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9ee0003e00 con 0x55a23fee08b0 2024-04-06T14:11:03.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9f07138b80 10 client.16365 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.176+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f9eec009af0 con 0x55a23fee08b0 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ee006a080 con 0x55a23fee08b0 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9ee005c030 con 0x55a23fee08b0 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 10 client.16365 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 10 client.16365 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 20 client.16365 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23ff46120 0x55a23ff48540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 4 client.16365 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.177+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55a23fc77cf0 con 0x55a23ff46120 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.178+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23ff46120 0x55a23ff48540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9ef8d20700 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23ff46120 0x55a23ff48540 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=84a4858c87e7d3b7 server_cookie=5259f9a7d8c47da2 in_seq=0 out_seq=0 2024-04-06T14:11:03.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9ed3fff700 10 client.16365 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9ed3fff700 1 -- 172.21.15.90:0/1354797850 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55a23fc77cf0 con 0x55a23ff46120 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9ed3fff700 10 client.16365 handle_command_reply: tid=0 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 client.16365 shutdown 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 msgr2=0x55a23feeeec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 secure :-1 s=READY pgs=578 cs=0 l=1 rev1=1 crypto rx=0x7f9ee00347c0 tx=0x7f9ee0054640 comp rx=0 tx=0).stop 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 shutdown_connections 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23ff46120 0x55a23ff48540 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23feea5f0 0x55a23feef440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9ed3fff700 0 client.16365 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23fedff90 0x55a23feee980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 --2- 172.21.15.90:0/1354797850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23fee08b0 0x55a23feeeec0 unknown :-1 s=CLOSED pgs=578 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 >> 172.21.15.90:0/1354797850 conn(0x55a23fe366a0 msgr2=0x55a23fedef80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 shutdown_connections 2024-04-06T14:11:03.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 1 -- 172.21.15.90:0/1354797850 wait complete. 2024-04-06T14:11:03.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.179+0000 7f9f07138b80 20 client.16365 trim_cache size 0 max 16384 2024-04-06T14:11:03.186 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 msgr2=0x7f9ee40a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 0x7f9ee40a3750 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f9ef406bff0 tx=0x7f9ee0036040 comp rx=0 tx=0).stop 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 msgr2=0x7f9ef40728b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f9ee8004760 tx=0x7f9ee8004790 comp rx=0 tx=0).stop 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 shutdown_connections 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9ee40a1290 0x7f9ee40a3750 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9ef406ae00 0x7f9ef406b2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9ef40ffe70 0x7f9ef40728b0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 --2- 172.21.15.90:0/1140083780 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9ef4104f80 0x7f9ef406a8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:03.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 >> 172.21.15.90:0/1140083780 conn(0x7f9ef4076d40 msgr2=0x7f9ef40694a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:03.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 shutdown_connections 2024-04-06T14:11:03.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:03.180+0000 7f9efaf84700 1 -- 172.21.15.90:0/1140083780 wait complete. 2024-04-06T14:11:03.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:02 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/501913579' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:04.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:03 smithi134 ceph-mon[47549]: pgmap v173: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 17 MiB/s wr, 843 op/s 2024-04-06T14:11:04.197 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:11:04.198 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:11:04.198 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:11:04.227 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:03 smithi090 ceph-mon[43420]: pgmap v173: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 17 MiB/s wr, 843 op/s 2024-04-06T14:11:04.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:03 smithi105 ceph-mon[46969]: pgmap v173: 97 pgs: 97 active+clean; 7.4 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 17 MiB/s wr, 843 op/s 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.383+0000 7f08a8476700 1 -- 172.21.15.90:0/162889124 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 msgr2=0x7f08a010c2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.383+0000 7f08a8476700 1 --2- 172.21.15.90:0/162889124 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a010c2d0 secure :-1 s=READY pgs=579 cs=0 l=1 rev1=1 crypto rx=0x7f089800b0a0 tx=0x7f089800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.384+0000 7f08a8476700 1 -- 172.21.15.90:0/162889124 shutdown_connections 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.384+0000 7f08a8476700 1 --2- 172.21.15.90:0/162889124 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a010c2d0 unknown :-1 s=CLOSED pgs=579 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.384+0000 7f08a8476700 1 --2- 172.21.15.90:0/162889124 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 0x7f08a01000b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.384+0000 7f08a8476700 1 --2- 172.21.15.90:0/162889124 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a0100680 0x7f08a0104040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.384+0000 7f08a8476700 1 -- 172.21.15.90:0/162889124 >> 172.21.15.90:0/162889124 conn(0x7f08a0074b50 msgr2=0x7f08a0074f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.385+0000 7f08a8476700 1 -- 172.21.15.90:0/162889124 shutdown_connections 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.385+0000 7f08a8476700 1 -- 172.21.15.90:0/162889124 wait complete. 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 Processor -- start 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 -- start start 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 0x7f08a01994c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a019dda0 0x7f08a019e220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08a019e820 con 0x7f08a0104720 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08a019e990 con 0x7f08a00ffcd0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a8476700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08a019eb50 con 0x7f08a019dda0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42738/0 (socket says 172.21.15.90:42738) 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 -- 172.21.15.90:0/475965746 learned_addr learned my addr 172.21.15.90:0/475965746 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a6212700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 0x7f08a01994c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a6a13700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a019dda0 0x7f08a019e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 -- 172.21.15.90:0/475965746 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a019dda0 msgr2=0x7f08a019e220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a019dda0 0x7f08a019e220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 -- 172.21.15.90:0/475965746 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 msgr2=0x7f08a01994c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 0x7f08a01994c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.386+0000 7f08a5a11700 1 -- 172.21.15.90:0/475965746 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0898009d00 con 0x7f08a0104720 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08a5a11700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f089000e940 tx=0x7f089000ec50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08977fe700 1 -- 172.21.15.90:0/475965746 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0890009730 con 0x7f08a0104720 2024-04-06T14:11:04.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08a01a60b0 con 0x7f08a0104720 2024-04-06T14:11:04.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08977fe700 1 -- 172.21.15.90:0/475965746 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0890010430 con 0x7f08a0104720 2024-04-06T14:11:04.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08977fe700 1 -- 172.21.15.90:0/475965746 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089000cea0 con 0x7f08a0104720 2024-04-06T14:11:04.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.387+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08a01a6590 con 0x7f08a0104720 2024-04-06T14:11:04.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.391+0000 7f08b462ab80 1 Processor -- start 2024-04-06T14:11:04.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.392+0000 7f08977fe700 1 -- 172.21.15.90:0/475965746 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0890018680 con 0x7f08a0104720 2024-04-06T14:11:04.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.392+0000 7f08977fe700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 0x7f088c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08977fe700 1 -- 172.21.15.90:0/475965746 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f0890014070 con 0x7f08a0104720 2024-04-06T14:11:04.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 -- start start 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08a6a13700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 0x7f088c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b59120 0x556b75b62920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b65250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75ab0a10 con 0x556b75b58800 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75aae3d0 con 0x556b75b62e60 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08b462ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75aa9cc0 con 0x556b75b59120 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.393+0000 7f08a6212700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42744/0 (socket says 172.21.15.90:42744) 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 -- 172.21.15.90:0/3086724234 learned_addr learned my addr 172.21.15.90:0/3086724234 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6a13700 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b65250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a5a11700 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b59120 0x556b75b62920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6a13700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 0x7f088c0a3750 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f089800b780 tx=0x7f0898006230 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 -- 172.21.15.90:0/3086724234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b59120 msgr2=0x556b75b62920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b59120 0x556b75b62920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 -- 172.21.15.90:0/3086724234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 msgr2=0x556b75b65250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b65250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 -- 172.21.15.90:0/3086724234 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556b7592b430 con 0x556b75b58800 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f08a6212700 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f089c009d80 tx=0x7f089c011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bbb80a70f08da568 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f0894da5700 1 -- 172.21.15.90:0/3086724234 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c004230 con 0x556b75b58800 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f0894da5700 1 -- 172.21.15.90:0/3086724234 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f089c004390 con 0x556b75b58800 2024-04-06T14:11:04.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.394+0000 7f0894da5700 1 -- 172.21.15.90:0/3086724234 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c019860 con 0x556b75b58800 2024-04-06T14:11:04.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 -- 172.21.15.90:0/3086724234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 msgr2=0x556b75b58be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 secure :-1 s=READY pgs=581 cs=0 l=1 rev1=1 crypto rx=0x7f089c009d80 tx=0x7f089c011720 comp rx=0 tx=0).stop 2024-04-06T14:11:04.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 -- 172.21.15.90:0/3086724234 shutdown_connections 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b65250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b58be0 unknown :-1 s=CLOSED pgs=581 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 --2- 172.21.15.90:0/3086724234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b59120 0x556b75b62920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 -- 172.21.15.90:0/3086724234 >> 172.21.15.90:0/3086724234 conn(0x556b75ab7720 msgr2=0x556b75b53b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 -- 172.21.15.90:0/3086724234 shutdown_connections 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.395+0000 7f08b462ab80 1 -- 172.21.15.90:0/3086724234 wait complete. 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 -- start start 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 Processor -- start 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b5fbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b601e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75a45e00 con 0x556b75b59120 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75a45aa0 con 0x556b75b62e60 2024-04-06T14:11:04.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b7592d720 con 0x556b75b58800 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08a6212700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08b462ab80 10 client.0 _ll_register_callbacks cb 0x556b75a4f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08a6212700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51118/0 (socket says 172.21.15.90:51118) 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.396+0000 7f08a6212700 1 -- 172.21.15.90:0/2248691531 learned_addr learned my addr 172.21.15.90:0/2248691531 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b5fbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 -- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 msgr2=0x556b75b601e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b601e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 -- 172.21.15.90:0/2248691531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 msgr2=0x556b75b5fbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b5fbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 -- 172.21.15.90:0/2248691531 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f089c0099e0 con 0x556b75b58800 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b5fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08a6212700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f089c011a70 tx=0x7f089c003940 comp rx=0 tx=0).ready entity=mon.1 client_cookie=94881d2896468b1b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f0889ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:04.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c005450 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f089c004500 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x556b75a6a400 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f089c003da0 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08b462ab80 10 client.26002 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.397+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x556b758d7be0 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x556b75a6a400 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f089c021040 con 0x556b75b58800 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 10 client.26002 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 10 client.26002 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 20 client.26002 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75b72970 0x556b75b74d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 4 client.26002 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:04.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.398+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x556b75aa2780 con 0x556b75b72970 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.399+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75b72970 0x556b75b74d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.399+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75b72970 0x556b75b74d60 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=48e73fc3687c12b server_cookie=38c2802fefd25b65 in_seq=0 out_seq=0 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.399+0000 7f0889ffb700 10 client.26002 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f0889ffb700 1 -- 172.21.15.90:0/2248691531 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x556b75aa2780 con 0x556b75b72970 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f0889ffb700 10 client.26002 handle_command_reply: tid=0 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 client.26002 shutdown 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 msgr2=0x556b75b62870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f089c011a70 tx=0x7f089c003940 comp rx=0 tx=0).stop 2024-04-06T14:11:04.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 shutdown_connections 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b62e60 0x556b75b601e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75b72970 0x556b75b74d60 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b58800 0x556b75b62870 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2248691531 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b5fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 >> 172.21.15.90:0/2248691531 conn(0x556b75ab7720 msgr2=0x556b75b52ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f0889ffb700 0 client.26002 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 shutdown_connections 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.400+0000 7f08b462ab80 1 -- 172.21.15.90:0/2248691531 wait complete. 2024-04-06T14:11:04.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.401+0000 7f08b462ab80 20 client.26002 trim_cache size 0 max 16384 2024-04-06T14:11:04.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.423+0000 7f08b462ab80 1 Processor -- start 2024-04-06T14:11:04.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.423+0000 7f08b462ab80 1 -- start start 2024-04-06T14:11:04.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b69ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75b53420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75b6a020 con 0x556b75b58800 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75b5ff30 con 0x556b75b59120 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08b462ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75b6ee80 con 0x556b75b62e60 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a5a11700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b69ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a5a11700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54044/0 (socket says 172.21.15.90:54044) 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a5a11700 1 -- 172.21.15.90:0/1497820041 learned_addr learned my addr 172.21.15.90:0/1497820041 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6a13700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75b53420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 -- 172.21.15.90:0/1497820041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 msgr2=0x556b75b53420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75b53420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 -- 172.21.15.90:0/1497820041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 msgr2=0x556b75b6e940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 -- 172.21.15.90:0/1497820041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556b75b4fbe0 con 0x556b75b58800 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6a13700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75b53420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a5a11700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.424+0000 7f08a6212700 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b69ae0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f089c009cd0 tx=0x7f089c005a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52bbfd009ccfbf1b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08955a6700 1 -- 172.21.15.90:0/1497820041 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c019730 con 0x556b75b58800 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08955a6700 1 -- 172.21.15.90:0/1497820041 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f089c015070 con 0x556b75b58800 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08955a6700 1 -- 172.21.15.90:0/1497820041 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c019bd0 con 0x556b75b58800 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 -- 172.21.15.90:0/1497820041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 msgr2=0x556b75b69ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b69ae0 secure :-1 s=READY pgs=582 cs=0 l=1 rev1=1 crypto rx=0x7f089c009cd0 tx=0x7f089c005a00 comp rx=0 tx=0).stop 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 -- 172.21.15.90:0/1497820041 shutdown_connections 2024-04-06T14:11:04.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75b53420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b58800 0x556b75b69ae0 unknown :-1 s=CLOSED pgs=582 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 --2- 172.21.15.90:0/1497820041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b59120 0x556b75b6e940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 -- 172.21.15.90:0/1497820041 >> 172.21.15.90:0/1497820041 conn(0x556b75ab7720 msgr2=0x556b75b6e230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 -- 172.21.15.90:0/1497820041 shutdown_connections 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.425+0000 7f08b462ab80 1 -- 172.21.15.90:0/1497820041 wait complete. 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 -- start start 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 Processor -- start 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b68ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75bc0770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75bc0cb0 con 0x556b75b59120 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75bc0e20 con 0x556b75b58800 2024-04-06T14:11:04.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x556b75bc0fe0 con 0x556b75b62e60 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08b462ab80 10 client.0 _ll_register_callbacks cb 0x556b75a4f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08a6212700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08a6212700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54046/0 (socket says 172.21.15.90:54046) 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.426+0000 7f08a6212700 1 -- 172.21.15.90:0/2488444425 learned_addr learned my addr 172.21.15.90:0/2488444425 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 -- 172.21.15.90:0/2488444425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 msgr2=0x556b75bc0770 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75bc0770 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 -- 172.21.15.90:0/2488444425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 msgr2=0x556b75b68ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b68ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 -- 172.21.15.90:0/2488444425 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556b758d7be0 con 0x556b75b58800 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f08a6212700 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f089c003980 tx=0x7f089c001e70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fe7e587e37d22740 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f0889ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089c019930 con 0x556b75b58800 2024-04-06T14:11:04.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.427+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f089c019a90 con 0x556b75b58800 2024-04-06T14:11:04.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.428+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f089c004390 con 0x556b75b58800 2024-04-06T14:11:04.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f089c003da0 con 0x556b75b58800 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 10 client.26102 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x556b75a6ff20 con 0x556b75b58800 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f089c004390 con 0x556b75b58800 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f089c046020 con 0x556b75b58800 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 10 client.26102 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 10 client.26102 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 20 client.26102 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75bc8380 0x556b75bca770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 4 client.26102 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x556b758f8d70 con 0x556b75bc8380 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.429+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75bc8380 0x556b75bca770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.430+0000 7f0889ffb700 10 client.26102 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.430+0000 7f08a5a11700 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75bc8380 0x556b75bca770 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6f9ed7d38c5d5860 server_cookie=38e9e8652589db85 in_seq=0 out_seq=0 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.431+0000 7f0889ffb700 1 -- 172.21.15.90:0/2488444425 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x556b758f8d70 con 0x556b75bc8380 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.431+0000 7f0889ffb700 10 client.26102 handle_command_reply: tid=0 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.431+0000 7f08b462ab80 1 client.26102 shutdown 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.431+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 msgr2=0x556b75b68ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.431+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f089c003980 tx=0x7f089c001e70 comp rx=0 tx=0).stop 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 shutdown_connections 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x556b75bc8380 0x556b75bca770 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x556b75b62e60 0x556b75bc0770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x556b75b58800 0x556b75b68ab0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f0889ffb700 0 client.26102 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 --2- 172.21.15.90:0/2488444425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x556b75b59120 0x556b75b68ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 >> 172.21.15.90:0/2488444425 conn(0x556b75ab7720 msgr2=0x556b75b577f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 shutdown_connections 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 1 -- 172.21.15.90:0/2488444425 wait complete. 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08b462ab80 20 client.26102 trim_cache size 0 max 16384 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "567f9a34-744a-4ca7-ab0c-1dd9958a6a12", 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:11:04.436 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 msgr2=0x7f088c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 0x7f088c0a3750 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f089800b780 tx=0x7f0898006230 comp rx=0 tx=0).stop 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 msgr2=0x7f08a0199a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.432+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 secure :-1 s=READY pgs=580 cs=0 l=1 rev1=1 crypto rx=0x7f089000e940 tx=0x7f089000ec50 comp rx=0 tx=0).stop 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 shutdown_connections 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f088c0a1290 0x7f088c0a3750 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08a019dda0 0x7f08a019e220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08a00ffcd0 0x7f08a01994c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 --2- 172.21.15.90:0/475965746 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08a0104720 0x7f08a0199a00 unknown :-1 s=CLOSED pgs=580 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.433+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 >> 172.21.15.90:0/475965746 conn(0x7f08a0074b50 msgr2=0x7f08a010af00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.434+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 shutdown_connections 2024-04-06T14:11:04.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.434+0000 7f08a8476700 1 -- 172.21.15.90:0/475965746 wait complete. 2024-04-06T14:11:04.452 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:11:04.452 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:11:04.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.709+0000 7f660d0f2700 1 -- 172.21.15.90:0/2553427601 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660810c6b0 msgr2=0x7f6608114230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.709+0000 7f660d0f2700 1 --2- 172.21.15.90:0/2553427601 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660810c6b0 0x7f6608114230 secure :-1 s=READY pgs=583 cs=0 l=1 rev1=1 crypto rx=0x7f65f8009990 tx=0x7f65f8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.710+0000 7f660d0f2700 1 -- 172.21.15.90:0/2553427601 shutdown_connections 2024-04-06T14:11:04.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.710+0000 7f660d0f2700 1 --2- 172.21.15.90:0/2553427601 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660810c6b0 0x7f6608114230 unknown :-1 s=CLOSED pgs=583 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.710+0000 7f660d0f2700 1 --2- 172.21.15.90:0/2553427601 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f6608107970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.710+0000 7f660d0f2700 1 --2- 172.21.15.90:0/2553427601 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6608107f40 0x7f660810bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.710+0000 7f660d0f2700 1 -- 172.21.15.90:0/2553427601 >> 172.21.15.90:0/2553427601 conn(0x7f66080752a0 msgr2=0x7f66080776c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.714+0000 7f660d0f2700 1 -- 172.21.15.90:0/2553427601 shutdown_connections 2024-04-06T14:11:04.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.714+0000 7f660d0f2700 1 -- 172.21.15.90:0/2553427601 wait complete. 2024-04-06T14:11:04.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.715+0000 7f660d0f2700 1 Processor -- start 2024-04-06T14:11:04.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 -- start start 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f66081a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 0x7f66081a3b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 0x7f660819dc80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f660819e250 con 0x7f660819d7d0 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f660819e390 con 0x7f6608107590 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.716+0000 7f660d0f2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f660819e550 con 0x7f660810c6b0 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 0x7f66081a3b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f6606d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f66081a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 0x7f66081a3b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51154/0 (socket says 172.21.15.90:51154) 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f6606d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f66081a3650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54048/0 (socket says 172.21.15.90:54048) 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660659c700 1 -- 172.21.15.90:0/1546508802 learned_addr learned my addr 172.21.15.90:0/1546508802 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 0x7f660819dc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 -- 172.21.15.90:0/1546508802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 msgr2=0x7f66081a3b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 0x7f66081a3b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 -- 172.21.15.90:0/1546508802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 msgr2=0x7f66081a3650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f66081a3650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 -- 172.21.15.90:0/1546508802 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65f80095f0 con 0x7f660819d7d0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f660759e700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 0x7f660819dc80 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f65f8009990 tx=0x7f65f8005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.717+0000 7f65effff700 1 -- 172.21.15.90:0/1546508802 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f801d070 con 0x7f660819d7d0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.718+0000 7f65effff700 1 -- 172.21.15.90:0/1546508802 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65f800fb30 con 0x7f660819d7d0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.718+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66081aa460 con 0x7f660819d7d0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.718+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66081aa890 con 0x7f660819d7d0 2024-04-06T14:11:04.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.719+0000 7f65effff700 1 -- 172.21.15.90:0/1546508802 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f8022be0 con 0x7f660819d7d0 2024-04-06T14:11:04.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.722+0000 7f65effff700 1 -- 172.21.15.90:0/1546508802 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f65f800f670 con 0x7f660819d7d0 2024-04-06T14:11:04.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.723+0000 7f65effff700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 0x7f65f00a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.723+0000 7f65effff700 1 -- 172.21.15.90:0/1546508802 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f65f80ce240 con 0x7f660819d7d0 2024-04-06T14:11:04.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.723+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 0x7f65f00a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.724+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 0x7f65f00a3810 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f660000b450 tx=0x7f660000d040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 Processor -- start 2024-04-06T14:11:04.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 -- start start 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717d780 0x55d7d717db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d71878a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 0x55d7d718a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d70d5990 con 0x55d7d7187de0 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d70d3350 con 0x55d7d717d780 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f66192a6b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d70cec40 con 0x55d7d717e0a0 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d71878a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f660759e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 0x55d7d718a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d71878a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51166/0 (socket says 172.21.15.90:51166) 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.726+0000 7f660659c700 1 -- 172.21.15.90:0/2230457257 learned_addr learned my addr 172.21.15.90:0/2230457257 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 -- 172.21.15.90:0/2230457257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 msgr2=0x55d7d71878a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d71878a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 -- 172.21.15.90:0/2230457257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717d780 msgr2=0x55d7d717db60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717d780 0x55d7d717db60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 -- 172.21.15.90:0/2230457257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d7d6f503b0 con 0x55d7d7187de0 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f660759e700 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 0x55d7d718a1d0 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f65f80cb960 tx=0x7f65f8022680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84014d4652ed7b2e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f65ed7fa700 1 -- 172.21.15.90:0/2230457257 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f809b4f0 con 0x55d7d7187de0 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f65ed7fa700 1 -- 172.21.15.90:0/2230457257 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65f809f320 con 0x55d7d7187de0 2024-04-06T14:11:04.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.727+0000 7f65ed7fa700 1 -- 172.21.15.90:0/2230457257 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f809b750 con 0x55d7d7187de0 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.728+0000 7f66192a6b80 1 -- 172.21.15.90:0/2230457257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 msgr2=0x55d7d718a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.728+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 0x55d7d718a1d0 secure :-1 s=READY pgs=585 cs=0 l=1 rev1=1 crypto rx=0x7f65f80cb960 tx=0x7f65f8022680 comp rx=0 tx=0).stop 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 -- 172.21.15.90:0/2230457257 shutdown_connections 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d7187de0 0x55d7d718a1d0 unknown :-1 s=CLOSED pgs=585 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717d780 0x55d7d717db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2230457257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d71878a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 -- 172.21.15.90:0/2230457257 >> 172.21.15.90:0/2230457257 conn(0x55d7d70dc6a0 msgr2=0x55d7d7178a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 -- 172.21.15.90:0/2230457257 shutdown_connections 2024-04-06T14:11:04.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 -- 172.21.15.90:0/2230457257 wait complete. 2024-04-06T14:11:04.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 -- start start 2024-04-06T14:11:04.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.729+0000 7f66192a6b80 1 Processor -- start 2024-04-06T14:11:04.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d71878a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d71852d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d706ad80 con 0x55d7d717d780 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d706aa20 con 0x55d7d7187de0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d6f526a0 con 0x55d7d717e0a0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f66192a6b80 10 client.0 _ll_register_callbacks cb 0x55d7d7074470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51180/0 (socket says 172.21.15.90:51180) 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 -- 172.21.15.90:0/2112383454 learned_addr learned my addr 172.21.15.90:0/2112383454 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f6606d9d700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d71878a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660759e700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d71852d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 -- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 msgr2=0x55d7d71852d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d71852d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 -- 172.21.15.90:0/2112383454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 msgr2=0x55d7d71878a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d71878a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.730+0000 7f660659c700 1 -- 172.21.15.90:0/2112383454 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66081aa890 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f660659c700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f65fc006b00 tx=0x7f65fc0108d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8840f1b61e7857ea server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f65eaff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65fc014640 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f65fc0147a0 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d7d708f380 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f65fc014960 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f66192a6b80 10 client.26011 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.731+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d7d6efcb60 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d7d708f380 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f65fc01dc10 con 0x55d7d717e0a0 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 10 client.26011 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 10 client.26011 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:04.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 20 client.26011 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:04.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71977c0 0x55d7d7199be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 4 client.26011 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:04.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d7d70c7700 con 0x55d7d71977c0 2024-04-06T14:11:04.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.732+0000 7f6606d9d700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71977c0 0x55d7d7199be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.733+0000 7f65eaff5700 10 client.26011 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.733+0000 7f6606d9d700 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71977c0 0x55d7d7199be0 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3fe0518640902933 server_cookie=22b17673d6e9fec5 in_seq=0 out_seq=0 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f65eaff5700 1 -- 172.21.15.90:0/2112383454 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d7d70c7700 con 0x55d7d71977c0 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f65eaff5700 10 client.26011 handle_command_reply: tid=0 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 client.26011 shutdown 2024-04-06T14:11:04.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 msgr2=0x55d7d7184b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f65fc006b00 tx=0x7f65fc0108d0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 shutdown_connections 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d71852d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d71878a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71977c0 0x55d7d7199be0 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 --2- 172.21.15.90:0/2112383454 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717e0a0 0x55d7d7184b90 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.734+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 >> 172.21.15.90:0/2112383454 conn(0x55d7d70dc6a0 msgr2=0x55d7d7177c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.735+0000 7f65eaff5700 0 client.26011 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.735+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 shutdown_connections 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.735+0000 7f66192a6b80 1 -- 172.21.15.90:0/2112383454 wait complete. 2024-04-06T14:11:04.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.735+0000 7f66192a6b80 20 client.26011 trim_cache size 0 max 16384 2024-04-06T14:11:04.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 Processor -- start 2024-04-06T14:11:04.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 -- start start 2024-04-06T14:11:04.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717e0a0 0x55d7d71783a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 0x55d7d71788e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d717ea10 con 0x55d7d717d780 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d7184f10 con 0x55d7d717e0a0 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.761+0000 7f66192a6b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d717fa50 con 0x55d7d7187de0 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f6606d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 0x55d7d71788e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f6606d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42828/0 (socket says 172.21.15.90:42828) 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f6606d9d700 1 -- 172.21.15.90:0/3968370484 learned_addr learned my addr 172.21.15.90:0/3968370484 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660659c700 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717e0a0 0x55d7d71783a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 -- 172.21.15.90:0/3968370484 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717e0a0 msgr2=0x55d7d71783a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717e0a0 0x55d7d71783a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 -- 172.21.15.90:0/3968370484 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 msgr2=0x55d7d718e7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 -- 172.21.15.90:0/3968370484 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d7d7094ea0 con 0x55d7d7187de0 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f6606d9d700 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:04.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f660759e700 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 0x55d7d71788e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x55d7d717f1f0 tx=0x7f65f801a040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=73755300e9653ff7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f65edffb700 1 -- 172.21.15.90:0/3968370484 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f8065070 con 0x55d7d7187de0 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f65edffb700 1 -- 172.21.15.90:0/3968370484 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65f80b9030 con 0x55d7d7187de0 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.762+0000 7f65edffb700 1 -- 172.21.15.90:0/3968370484 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65f8095e20 con 0x55d7d7187de0 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 -- 172.21.15.90:0/3968370484 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 msgr2=0x55d7d71788e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 0x55d7d71788e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x55d7d717f1f0 tx=0x7f65f801a040 comp rx=0 tx=0).stop 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 -- 172.21.15.90:0/3968370484 shutdown_connections 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d7187de0 0x55d7d71788e0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717d780 0x55d7d718e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 --2- 172.21.15.90:0/3968370484 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d717e0a0 0x55d7d71783a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 -- 172.21.15.90:0/3968370484 >> 172.21.15.90:0/3968370484 conn(0x55d7d70dc6a0 msgr2=0x55d7d717ff40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 -- 172.21.15.90:0/3968370484 shutdown_connections 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.763+0000 7f66192a6b80 1 -- 172.21.15.90:0/3968370484 wait complete. 2024-04-06T14:11:04.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 -- start start 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 Processor -- start 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 0x55d7d718e620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d718d100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d718d640 con 0x55d7d717e0a0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d718d7b0 con 0x55d7d7187de0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d7d718d920 con 0x55d7d717d780 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f66192a6b80 10 client.0 _ll_register_callbacks cb 0x55d7d7074470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f660659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f660659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42832/0 (socket says 172.21.15.90:42832) 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f660659c700 1 -- 172.21.15.90:0/1852940592 learned_addr learned my addr 172.21.15.90:0/1852940592 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 0x55d7d718e620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.764+0000 7f660759e700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d718d100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 -- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 msgr2=0x55d7d718d100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d718d100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 -- 172.21.15.90:0/1852940592 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 msgr2=0x55d7d718cbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 -- 172.21.15.90:0/1852940592 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f660000be10 con 0x55d7d717d780 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f660659c700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f6606d9d700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 0x55d7d718e620 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f6600009b00 tx=0x7f6600004b80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=baafc928eac2ed89 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:04.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f65eaff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6600010040 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.765+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f6600016a80 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f65f809f320 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6600016c40 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 10 client.26017 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d7d6f503b0 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65f809f320 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6600023400 con 0x55d7d717d780 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 10 client.26017 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 10 client.26017 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 20 client.26017 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:04.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71ecbc0 0x55d7d71eefe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 4 client.26017 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.766+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d7d6f1dcf0 con 0x55d7d71ecbc0 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f660659c700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71ecbc0 0x55d7d71eefe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f65eaff5700 10 client.26017 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f660659c700 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71ecbc0 0x55d7d71eefe0 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=71c7944284909fad server_cookie=db82102201dd817a in_seq=0 out_seq=0 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f65eaff5700 1 -- 172.21.15.90:0/1852940592 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55d7d6f1dcf0 con 0x55d7d71ecbc0 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f65eaff5700 10 client.26017 handle_command_reply: tid=0 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.767+0000 7f66192a6b80 1 client.26017 shutdown 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 msgr2=0x55d7d718e620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 0x55d7d718e620 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f6600009b00 tx=0x7f6600004b80 comp rx=0 tx=0).stop 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 shutdown_connections 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d7d7187de0 0x55d7d718d100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d7d71ecbc0 0x55d7d71eefe0 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d7d717d780 0x55d7d718e620 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 --2- 172.21.15.90:0/1852940592 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d7d717e0a0 0x55d7d718cbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 >> 172.21.15.90:0/1852940592 conn(0x55d7d70dc6a0 msgr2=0x55d7d7183460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f65eaff5700 0 client.26017 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 shutdown_connections 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.768+0000 7f66192a6b80 1 -- 172.21.15.90:0/1852940592 wait complete. 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.769+0000 7f66192a6b80 20 client.26017 trim_cache size 0 max 16384 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:04.772 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: "567f9a34-744a-4ca7-ab0c-1dd9958a6a12": { 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "567f9a34-744a-4ca7-ab0c-1dd9958a6a12", 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:11:04.773 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 msgr2=0x7f65f00a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 0x7f65f00a3810 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f660000b450 tx=0x7f660000d040 comp rx=0 tx=0).stop 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 msgr2=0x7f660819dc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 0x7f660819dc80 secure :-1 s=READY pgs=584 cs=0 l=1 rev1=1 crypto rx=0x7f65f8009990 tx=0x7f65f8005740 comp rx=0 tx=0).stop 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 shutdown_connections 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65f00a1350 0x7f65f00a3810 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f660819d7d0 0x7f660819dc80 unknown :-1 s=CLOSED pgs=584 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6608107590 0x7f66081a3650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 --2- 172.21.15.90:0/1546508802 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f660810c6b0 0x7f66081a3b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:04.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 >> 172.21.15.90:0/1546508802 conn(0x7f66080752a0 msgr2=0x7f6608077570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:04.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 shutdown_connections 2024-04-06T14:11:04.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:04.770+0000 7f660d0f2700 1 -- 172.21.15.90:0/1546508802 wait complete. 2024-04-06T14:11:04.790 INFO:tasks.cephfs.filesystem:scrub status for tag:567f9a34-744a-4ca7-ab0c-1dd9958a6a12 - {'path': '/', 'tag': '567f9a34-744a-4ca7-ab0c-1dd9958a6a12', 'options': 'recursive,force'} 2024-04-06T14:11:05.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:04 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:11:05.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:04 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:05.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:04 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:11:05.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:04 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:11:05.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:04 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:05.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:04 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:11:05.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:04 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:11:05.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:04 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:05.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:04 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:11:06.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:05 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:06.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:05 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:11:06.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:05 smithi134 ceph-mon[47549]: pgmap v174: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 46 MiB/s wr, 2.26k op/s 2024-04-06T14:11:06.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:05 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:06.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:05 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:11:06.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:05 smithi105 ceph-mon[46969]: pgmap v174: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 46 MiB/s wr, 2.26k op/s 2024-04-06T14:11:06.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:05 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:06.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:05 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:11:06.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:05 smithi090 ceph-mon[43420]: pgmap v174: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 46 MiB/s wr, 2.26k op/s 2024-04-06T14:11:08.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 320.0 s, 2177.6 tps, lat 45.762 ms stddev 54.493 2024-04-06T14:11:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:08 smithi105 ceph-mon[46969]: pgmap v175: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:09.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:08 smithi090 ceph-mon[43420]: pgmap v175: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:09.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:08 smithi134 ceph-mon[47549]: pgmap v175: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:10.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:10 smithi105 ceph-mon[46969]: pgmap v176: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:11.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:10 smithi090 ceph-mon[43420]: pgmap v176: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:11.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:10 smithi134 ceph-mon[47549]: pgmap v176: 97 pgs: 97 active+clean; 7.6 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 MiB/s wr, 1.85k op/s 2024-04-06T14:11:12.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:12 smithi105 ceph-mon[46969]: pgmap v177: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 49 MiB/s wr, 2.75k op/s 2024-04-06T14:11:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 325.0 s, 2277.4 tps, lat 43.723 ms stddev 68.552 2024-04-06T14:11:13.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:12 smithi090 ceph-mon[43420]: pgmap v177: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 49 MiB/s wr, 2.75k op/s 2024-04-06T14:11:13.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:12 smithi134 ceph-mon[47549]: pgmap v177: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 49 MiB/s wr, 2.75k op/s 2024-04-06T14:11:14.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:14 smithi105 ceph-mon[46969]: pgmap v178: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 42 MiB/s wr, 2.45k op/s 2024-04-06T14:11:15.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:14 smithi090 ceph-mon[43420]: pgmap v178: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 42 MiB/s wr, 2.45k op/s 2024-04-06T14:11:15.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:14 smithi134 ceph-mon[47549]: pgmap v178: 97 pgs: 97 active+clean; 7.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 42 MiB/s wr, 2.45k op/s 2024-04-06T14:11:16.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:16 smithi105 ceph-mon[46969]: pgmap v179: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 178 KiB/s rd, 50 MiB/s wr, 2.86k op/s 2024-04-06T14:11:17.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:16 smithi090 ceph-mon[43420]: pgmap v179: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 178 KiB/s rd, 50 MiB/s wr, 2.86k op/s 2024-04-06T14:11:17.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:16 smithi134 ceph-mon[47549]: pgmap v179: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 178 KiB/s rd, 50 MiB/s wr, 2.86k op/s 2024-04-06T14:11:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 330.0 s, 2604.8 tps, lat 38.772 ms stddev 47.212 2024-04-06T14:11:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:18 smithi105 ceph-mon[46969]: pgmap v180: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:18 smithi090 ceph-mon[43420]: pgmap v180: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:18 smithi134 ceph-mon[47549]: pgmap v180: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:21.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:20 smithi090 ceph-mon[43420]: pgmap v181: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:21.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:20 smithi134 ceph-mon[47549]: pgmap v181: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:21.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:20 smithi105 ceph-mon[46969]: pgmap v181: 97 pgs: 97 active+clean; 7.8 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 136 KiB/s rd, 21 MiB/s wr, 1.44k op/s 2024-04-06T14:11:23.029 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 335.0 s, 2878.4 tps, lat 34.596 ms stddev 39.477 2024-04-06T14:11:23.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:22 smithi090 ceph-mon[43420]: pgmap v182: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 150 KiB/s rd, 27 MiB/s wr, 1.73k op/s 2024-04-06T14:11:23.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:22 smithi134 ceph-mon[47549]: pgmap v182: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 150 KiB/s rd, 27 MiB/s wr, 1.73k op/s 2024-04-06T14:11:23.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:22 smithi105 ceph-mon[46969]: pgmap v182: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 150 KiB/s rd, 27 MiB/s wr, 1.73k op/s 2024-04-06T14:11:25.043 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:24 smithi090 ceph-mon[43420]: pgmap v183: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 14 MiB/s wr, 704 op/s 2024-04-06T14:11:25.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:24 smithi134 ceph-mon[47549]: pgmap v183: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 14 MiB/s wr, 704 op/s 2024-04-06T14:11:25.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:24 smithi105 ceph-mon[46969]: pgmap v183: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 14 MiB/s wr, 704 op/s 2024-04-06T14:11:27.050 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:26 smithi090 ceph-mon[43420]: pgmap v184: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 20 MiB/s wr, 1.08k op/s 2024-04-06T14:11:27.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:26 smithi134 ceph-mon[47549]: pgmap v184: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 20 MiB/s wr, 1.08k op/s 2024-04-06T14:11:27.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:26 smithi105 ceph-mon[46969]: pgmap v184: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 20 MiB/s wr, 1.08k op/s 2024-04-06T14:11:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 340.0 s, 2595.5 tps, lat 38.518 ms stddev 43.488 2024-04-06T14:11:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:28 smithi105 ceph-mon[46969]: pgmap v185: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:29.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:28 smithi090 ceph-mon[43420]: pgmap v185: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:29.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:28 smithi134 ceph-mon[47549]: pgmap v185: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:31.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:30 smithi090 ceph-mon[43420]: pgmap v186: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:31.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:30 smithi134 ceph-mon[47549]: pgmap v186: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:31.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:30 smithi105 ceph-mon[46969]: pgmap v186: 97 pgs: 97 active+clean; 7.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 12 MiB/s wr, 674 op/s 2024-04-06T14:11:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 345.0 s, 2334.4 tps, lat 42.832 ms stddev 47.901 2024-04-06T14:11:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:32 smithi090 ceph-mon[43420]: pgmap v187: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 18 MiB/s wr, 1.04k op/s 2024-04-06T14:11:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:11:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:11:33.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:32 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:11:33.148 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:32 smithi134 ceph-mon[47549]: pgmap v187: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 18 MiB/s wr, 1.04k op/s 2024-04-06T14:11:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:11:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:11:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:32 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:11:33.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:32 smithi105 ceph-mon[46969]: pgmap v187: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 18 MiB/s wr, 1.04k op/s 2024-04-06T14:11:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:11:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:11:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:32 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:11:34.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:33 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:11:34.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:33 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:11:34.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:33 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:11:34.790 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:11:34.791 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:11:34.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 -- 172.21.15.90:0/1681279126 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 msgr2=0x7f12cc0ff9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:34.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 --2- 172.21.15.90:0/1681279126 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc0ff9c0 secure :-1 s=READY pgs=586 cs=0 l=1 rev1=1 crypto rx=0x7f12bc009990 tx=0x7f12bc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:34.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 -- 172.21.15.90:0/1681279126 shutdown_connections 2024-04-06T14:11:34.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 --2- 172.21.15.90:0/1681279126 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc10c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:34.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 --2- 172.21.15.90:0/1681279126 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc0ff9c0 unknown :-1 s=CLOSED pgs=586 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:34.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 --2- 172.21.15.90:0/1681279126 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc0fff90 0x7f12cc103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:34.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.993+0000 7f12d24ea700 1 -- 172.21.15.90:0/1681279126 >> 172.21.15.90:0/1681279126 conn(0x7f12cc0761b0 msgr2=0x7f12cc0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:34.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.997+0000 7f12d24ea700 1 -- 172.21.15.90:0/1681279126 shutdown_connections 2024-04-06T14:11:34.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.997+0000 7f12d24ea700 1 -- 172.21.15.90:0/1681279126 wait complete. 2024-04-06T14:11:34.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.998+0000 7f12d24ea700 1 Processor -- start 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.998+0000 7f12d24ea700 1 -- start start 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc19beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 0x7f12cc1954d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12cc195c60 con 0x7f12cc0ff5e0 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12cc195dd0 con 0x7f12cc195020 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d24ea700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12cc199a40 con 0x7f12cc104700 2024-04-06T14:11:34.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 0x7f12cc1954d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52934/0 (socket says 172.21.15.90:52934) 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cb7fe700 1 -- 172.21.15.90:0/2321779755 learned_addr learned my addr 172.21.15.90:0/2321779755 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc19beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 -- 172.21.15.90:0/2321779755 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 msgr2=0x7f12cc19c430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 -- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 msgr2=0x7f12cc19beb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc19beb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 -- 172.21.15.90:0/2321779755 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12bc0095f0 con 0x7f12cc195020 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc19beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:34.999+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 0x7f12cc1954d0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f12c400be20 tx=0x7f12c4009ed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.000+0000 7f12c97fa700 1 -- 172.21.15.90:0/2321779755 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c400cc10 con 0x7f12cc195020 2024-04-06T14:11:35.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.000+0000 7f12c97fa700 1 -- 172.21.15.90:0/2321779755 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f12c400cd70 con 0x7f12cc195020 2024-04-06T14:11:35.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.000+0000 7f12c97fa700 1 -- 172.21.15.90:0/2321779755 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c40126c0 con 0x7f12cc195020 2024-04-06T14:11:35.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.000+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12cc199d20 con 0x7f12cc195020 2024-04-06T14:11:35.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.000+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12cc19a1e0 con 0x7f12cc195020 2024-04-06T14:11:35.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.002+0000 7f12c97fa700 1 -- 172.21.15.90:0/2321779755 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f12c4012860 con 0x7f12cc195020 2024-04-06T14:11:35.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.003+0000 7f12c97fa700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 0x7f12b40a3780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.003+0000 7f12c97fa700 1 -- 172.21.15.90:0/2321779755 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f12c4006e40 con 0x7f12cc195020 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 Processor -- start 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 -- start start 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaeab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaeb0a0 0x55cc3aaf48a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aa42990 con 0x55cc3aaf4de0 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aa40350 con 0x55cc3aaea780 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12de69eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aa3bc40 con 0x55cc3aaeb0a0 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59474/0 (socket says 172.21.15.90:59474) 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.004+0000 7f12d0a87700 1 -- 172.21.15.90:0/2560233659 learned_addr learned my addr 172.21.15.90:0/2560233659 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaeab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaeb0a0 0x55cc3aaf48a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 -- 172.21.15.90:0/2560233659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaeb0a0 msgr2=0x55cc3aaf48a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaeb0a0 0x55cc3aaf48a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 -- 172.21.15.90:0/2560233659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 msgr2=0x55cc3aaeab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaeab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 -- 172.21.15.90:0/2560233659 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cc3a8bd3b0 con 0x55cc3aaf4de0 2024-04-06T14:11:35.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaeab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.005+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f12c40a1630 tx=0x7f12c4099310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4aa18e1f3e6e451c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.006+0000 7f12b27fc700 1 -- 172.21.15.90:0/2560233659 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c409d510 con 0x55cc3aaf4de0 2024-04-06T14:11:35.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.006+0000 7f12b27fc700 1 -- 172.21.15.90:0/2560233659 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f12c40a1b60 con 0x55cc3aaf4de0 2024-04-06T14:11:35.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.006+0000 7f12b27fc700 1 -- 172.21.15.90:0/2560233659 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c4099c10 con 0x55cc3aaf4de0 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.007+0000 7f12de69eb80 1 -- 172.21.15.90:0/2560233659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 msgr2=0x55cc3aaf71d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.007+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 secure :-1 s=READY pgs=587 cs=0 l=1 rev1=1 crypto rx=0x7f12c40a1630 tx=0x7f12c4099310 comp rx=0 tx=0).stop 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 0x7f12b40a3780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12de69eb80 1 -- 172.21.15.90:0/2560233659 shutdown_connections 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf71d0 unknown :-1 s=CLOSED pgs=587 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaeab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2560233659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaeb0a0 0x55cc3aaf48a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12de69eb80 1 -- 172.21.15.90:0/2560233659 >> 172.21.15.90:0/2560233659 conn(0x55cc3aa496a0 msgr2=0x55cc3aae5a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.008+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 0x7f12b40a3780 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f12bc00c340 tx=0x7f12bc009ca0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.010+0000 7f12de69eb80 1 -- 172.21.15.90:0/2560233659 shutdown_connections 2024-04-06T14:11:35.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.010+0000 7f12de69eb80 1 -- 172.21.15.90:0/2560233659 wait complete. 2024-04-06T14:11:35.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 -- start start 2024-04-06T14:11:35.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 Processor -- start 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf2240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3a9d7d80 con 0x55cc3aaf2780 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3a9d7a20 con 0x55cc3aaea780 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3a8bf6a0 con 0x55cc3aaf4de0 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.011+0000 7f12de69eb80 10 client.0 _ll_register_callbacks cb 0x55cc3a9e1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12cbfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12cbfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60104/0 (socket says 172.21.15.90:60104) 2024-04-06T14:11:35.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12cbfff700 1 -- 172.21.15.90:0/2774663092 learned_addr learned my addr 172.21.15.90:0/2774663092 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 -- 172.21.15.90:0/2774663092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf4de0 msgr2=0x55cc3aaf2240 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf2240 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 -- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 msgr2=0x55cc3aaf1d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 -- 172.21.15.90:0/2774663092 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12bc01a040 con 0x55cc3aaf2780 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00ab0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x55cc3aaec1f0 tx=0x7f12c4095360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=225c7a00c12a2c30 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.012+0000 7f12ab7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c4098bd0 con 0x55cc3aaf2780 2024-04-06T14:11:35.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f12c4098d30 con 0x55cc3aaf2780 2024-04-06T14:11:35.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cc3a9fc380 con 0x55cc3aaf2780 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f12c4099940 con 0x55cc3aaf2780 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12de69eb80 10 client.16407 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f12cc19a1e0 con 0x55cc3aaf2780 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.013+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f12c407c020 con 0x55cc3aaf2780 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f12c40cc040 con 0x55cc3aaf2780 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 10 client.16407 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 10 client.16407 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 20 client.16407 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab04bb0 0x55cc3ab06fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 4 client.16407 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cc3aa34700 con 0x55cc3ab04bb0 2024-04-06T14:11:35.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.014+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab04bb0 0x55cc3ab06fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12ab7fe700 10 client.16407 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab04bb0 0x55cc3ab06fa0 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c885196e128001f8 server_cookie=c5d76d84ed5412df in_seq=0 out_seq=0 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2774663092 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cc3aa34700 con 0x55cc3ab04bb0 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12ab7fe700 10 client.16407 handle_command_reply: tid=0 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 client.16407 shutdown 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 msgr2=0x55cc3ab00ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00ab0 secure :-1 s=READY pgs=588 cs=0 l=1 rev1=1 crypto rx=0x55cc3aaec1f0 tx=0x7f12c4095360 comp rx=0 tx=0).stop 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 shutdown_connections 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00ab0 unknown :-1 s=CLOSED pgs=588 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf1d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.015+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab04bb0 0x55cc3ab06fa0 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2774663092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf2240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 >> 172.21.15.90:0/2774663092 conn(0x55cc3aa496a0 msgr2=0x55cc3aae4c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12ab7fe700 0 client.16407 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:35.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 shutdown_connections 2024-04-06T14:11:35.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12de69eb80 1 -- 172.21.15.90:0/2774663092 wait complete. 2024-04-06T14:11:35.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.016+0000 7f12de69eb80 20 client.16407 trim_cache size 0 max 16384 2024-04-06T14:11:35.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.039+0000 7f12de69eb80 1 Processor -- start 2024-04-06T14:11:35.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.039+0000 7f12de69eb80 1 -- start start 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaea780 0x55cc3ab00150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aaf4a60 con 0x55cc3aaea780 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aaf77a0 con 0x55cc3aaf4de0 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12de69eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3aaf4820 con 0x55cc3aaf2780 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60112/0 (socket says 172.21.15.90:60112) 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 -- 172.21.15.90:0/2891610689 learned_addr learned my addr 172.21.15.90:0/2891610689 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaea780 0x55cc3ab00150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 -- 172.21.15.90:0/2891610689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 msgr2=0x55cc3ab00690 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00690 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 -- 172.21.15.90:0/2891610689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaea780 msgr2=0x55cc3ab00150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaea780 0x55cc3ab00150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.040+0000 7f12d0a87700 1 -- 172.21.15.90:0/2891610689 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cc3a869b60 con 0x55cc3aaf4de0 2024-04-06T14:11:35.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.041+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f12c4095be0 tx=0x7f12c4004ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a5b3a9e855e38c38 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12b2ffd700 1 -- 172.21.15.90:0/2891610689 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c4099220 con 0x55cc3aaf4de0 2024-04-06T14:11:35.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12b2ffd700 1 -- 172.21.15.90:0/2891610689 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f12c407c030 con 0x55cc3aaf4de0 2024-04-06T14:11:35.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12b2ffd700 1 -- 172.21.15.90:0/2891610689 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c40a16d0 con 0x55cc3aaf4de0 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12de69eb80 1 -- 172.21.15.90:0/2891610689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 msgr2=0x55cc3aaeef60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f12c4095be0 tx=0x7f12c4004ad0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12de69eb80 1 -- 172.21.15.90:0/2891610689 shutdown_connections 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaeef60 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.042+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaea780 0x55cc3ab00150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2891610689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3ab00690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- 172.21.15.90:0/2891610689 >> 172.21.15.90:0/2891610689 conn(0x55cc3aa496a0 msgr2=0x55cc3aaf05d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- 172.21.15.90:0/2891610689 shutdown_connections 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- 172.21.15.90:0/2891610689 wait complete. 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- start start 2024-04-06T14:11:35.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 Processor -- start 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3aaf9810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf9d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3ab579b0 con 0x55cc3aaf4de0 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3ab57af0 con 0x55cc3aaea780 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.043+0000 7f12de69eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cc3ab57c60 con 0x55cc3aaf2780 2024-04-06T14:11:35.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12de69eb80 10 client.0 _ll_register_callbacks cb 0x55cc3a9e1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:35.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cbfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cbfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60122/0 (socket says 172.21.15.90:60122) 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cbfff700 1 -- 172.21.15.90:0/2832445617 learned_addr learned my addr 172.21.15.90:0/2832445617 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf9d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3aaf9810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 -- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 msgr2=0x55cc3aaf92d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 -- 172.21.15.90:0/2832445617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 msgr2=0x55cc3aaf9d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf9d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.044+0000 7f12cb7fe700 1 -- 172.21.15.90:0/2832445617 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12cc19a1e0 con 0x55cc3aaf2780 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12d0a87700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf9d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12cb7fe700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3aaf9810 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x55cc3aae6870 tx=0x7f12c0010a70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=57adf8b7de916db9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12ab7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:35.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f12c0009950 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f12c0014800 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f12c40a1b60 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f12c00149c0 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12de69eb80 10 client.26023 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.045+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cc3aa01ea0 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f12c40a1b60 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f12c001c3b0 con 0x55cc3aaf2780 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 10 client.26023 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 10 client.26023 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 20 client.26023 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab594f0 0x55cc3ab5b910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 4 client.26023 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55cc3a88acf0 con 0x55cc3ab594f0 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab594f0 0x55cc3ab5b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12ab7fe700 10 client.26023 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:35.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.046+0000 7f12cbfff700 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab594f0 0x55cc3ab5b910 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ca716159899a5084 server_cookie=945809456e33a24c in_seq=0 out_seq=0 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12ab7fe700 1 -- 172.21.15.90:0/2832445617 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55cc3a88acf0 con 0x55cc3ab594f0 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12ab7fe700 10 client.26023 handle_command_reply: tid=0 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 client.26023 shutdown 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 msgr2=0x55cc3aaf9810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3aaf9810 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x55cc3aae6870 tx=0x7f12c0010a70 comp rx=0 tx=0).stop 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 shutdown_connections 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cc3aaf4de0 0x55cc3aaf9d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cc3aaea780 0x55cc3aaf92d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cc3ab594f0 0x55cc3ab5b910 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 --2- 172.21.15.90:0/2832445617 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cc3aaf2780 0x55cc3aaf9810 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 >> 172.21.15.90:0/2832445617 conn(0x55cc3aa496a0 msgr2=0x55cc3aae9770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12ab7fe700 0 client.26023 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 shutdown_connections 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 1 -- 172.21.15.90:0/2832445617 wait complete. 2024-04-06T14:11:35.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.047+0000 7f12de69eb80 20 client.26023 trim_cache size 0 max 16384 2024-04-06T14:11:35.050 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:35.050 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:11:35.050 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:11:35.050 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 msgr2=0x7f12b40a3780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 0x7f12b40a3780 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f12bc00c340 tx=0x7f12bc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 msgr2=0x7f12cc1954d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 0x7f12cc1954d0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f12c400be20 tx=0x7f12c4009ed0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 shutdown_connections 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f12b40a12c0 0x7f12b40a3780 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f12cc195020 0x7f12cc1954d0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f12cc0ff5e0 0x7f12cc19beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 --2- 172.21.15.90:0/2321779755 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f12cc104700 0x7f12cc19c430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 >> 172.21.15.90:0/2321779755 conn(0x7f12cc0761b0 msgr2=0x7f12cc1029e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 shutdown_connections 2024-04-06T14:11:35.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.051+0000 7f12d24ea700 1 -- 172.21.15.90:0/2321779755 wait complete. 2024-04-06T14:11:35.064 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:11:35.064 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:11:35.097 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:34 smithi090 ceph-mon[43420]: pgmap v188: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 12 MiB/s wr, 742 op/s 2024-04-06T14:11:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:34 smithi134 ceph-mon[47549]: pgmap v188: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 12 MiB/s wr, 742 op/s 2024-04-06T14:11:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:34 smithi105 ceph-mon[46969]: pgmap v188: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 12 MiB/s wr, 742 op/s 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 -- 172.21.15.90:0/3299918489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc8111a50 msgr2=0x7f6cc8113e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 --2- 172.21.15.90:0/3299918489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc8111a50 0x7f6cc8113e40 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6cc0009f90 tx=0x7f6cc000b330 comp rx=0 tx=0).stop 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 -- 172.21.15.90:0/3299918489 shutdown_connections 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 --2- 172.21.15.90:0/3299918489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc8111a50 0x7f6cc8113e40 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 --2- 172.21.15.90:0/3299918489 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80fed30 0x7f6cc80ff110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 --2- 172.21.15.90:0/3299918489 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc80ff650 0x7f6cc8111510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.257+0000 7f6ccd622700 1 -- 172.21.15.90:0/3299918489 >> 172.21.15.90:0/3299918489 conn(0x7f6cc8075c80 msgr2=0x7f6cc80fd040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.258+0000 7f6ccd622700 1 -- 172.21.15.90:0/3299918489 shutdown_connections 2024-04-06T14:11:35.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.258+0000 7f6ccd622700 1 -- 172.21.15.90:0/3299918489 wait complete. 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.258+0000 7f6ccd622700 1 Processor -- start 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 -- start start 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 0x7f6cc810d2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80ff650 0x7f6cc810d830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 0x7f6cc81998c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cc819a050 con 0x7f6cc80ff650 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cc819a1c0 con 0x7f6cc80fed30 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.259+0000 7f6ccd622700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cc819a380 con 0x7f6cc8111a50 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.260+0000 7f6cc6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 0x7f6cc810d2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.260+0000 7f6cc77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 0x7f6cc81998c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.260+0000 7f6cc6ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 0x7f6cc810d2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60132/0 (socket says 172.21.15.90:60132) 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.260+0000 7f6cc6ffd700 1 -- 172.21.15.90:0/1734590897 learned_addr learned my addr 172.21.15.90:0/1734590897 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc67fc700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80ff650 0x7f6cc810d830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 -- 172.21.15.90:0/1734590897 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 msgr2=0x7f6cc810d2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 0x7f6cc810d2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 -- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80ff650 msgr2=0x7f6cc810d830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80ff650 0x7f6cc810d830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 -- 172.21.15.90:0/1734590897 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cc0009c40 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.261+0000 7f6cc77fe700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 0x7f6cc81998c0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6cc806a640 tx=0x7f6cc0009500 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.263+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cc000e040 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.263+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6cc0005040 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.263+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cc001c660 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.263+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6cc819a600 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.263+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6cc81c3cd0 con 0x7f6cc8111a50 2024-04-06T14:11:35.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.264+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6cc8101de0 con 0x7f6cc8111a50 2024-04-06T14:11:35.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.266+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6cc0005440 con 0x7f6cc8111a50 2024-04-06T14:11:35.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.266+0000 7f6caffff700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 0x7f6cb00a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.266+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f6cc00cd4f0 con 0x7f6cc8111a50 2024-04-06T14:11:35.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.267+0000 7f6cc6ffd700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 0x7f6cb00a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.268+0000 7f6cc6ffd700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 0x7f6cb00a3960 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f6cb801ad00 tx=0x7f6cb801a320 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.270+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6cc0096a20 con 0x7f6cc8111a50 2024-04-06T14:11:35.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.641+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6cc8100150 con 0x7f6cc8111a50 2024-04-06T14:11:35.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.642+0000 7f6caffff700 1 -- 172.21.15.90:0/1734590897 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f6cc009a040 con 0x7f6cc8111a50 2024-04-06T14:11:35.642 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:11:35.642 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:11:35.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 msgr2=0x7f6cb00a3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 0x7f6cb00a3960 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f6cb801ad00 tx=0x7f6cb801a320 comp rx=0 tx=0).stop 2024-04-06T14:11:35.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 msgr2=0x7f6cc81998c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 0x7f6cc81998c0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f6cc806a640 tx=0x7f6cc0009500 comp rx=0 tx=0).stop 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 shutdown_connections 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6cb00a14a0 0x7f6cb00a3960 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cc8111a50 0x7f6cc81998c0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cc80fed30 0x7f6cc810d2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 --2- 172.21.15.90:0/1734590897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cc80ff650 0x7f6cc810d830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.646+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 >> 172.21.15.90:0/1734590897 conn(0x7f6cc8075c80 msgr2=0x7f6cc80fd040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.648+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 shutdown_connections 2024-04-06T14:11:35.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.648+0000 7f6ccd622700 1 -- 172.21.15.90:0/1734590897 wait complete. 2024-04-06T14:11:35.651 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:11:35.664 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:35.664 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:11:35.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.875+0000 7f3031276700 1 -- 172.21.15.90:0/1188726405 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c0ffe70 msgr2=0x7f302c100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.875+0000 7f3031276700 1 --2- 172.21.15.90:0/1188726405 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c0ffe70 0x7f302c100250 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f301c0098f0 tx=0x7f301c009c00 comp rx=0 tx=0).stop 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 -- 172.21.15.90:0/1188726405 shutdown_connections 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 --2- 172.21.15.90:0/1188726405 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c10caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 --2- 172.21.15.90:0/1188726405 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c0ffe70 0x7f302c100250 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 --2- 172.21.15.90:0/1188726405 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c100820 0x7f302c104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 -- 172.21.15.90:0/1188726405 >> 172.21.15.90:0/1188726405 conn(0x7f302c076d50 msgr2=0x7f302c077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 -- 172.21.15.90:0/1188726405 shutdown_connections 2024-04-06T14:11:35.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 -- 172.21.15.90:0/1188726405 wait complete. 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.876+0000 7f3031276700 1 Processor -- start 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 -- start start 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 0x7f302c19de80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f302c19e480 con 0x7f302c0ffe70 2024-04-06T14:11:35.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f302c19e5f0 con 0x7f302c19d9d0 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f3031276700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f302c19e7b0 con 0x7f302c104f70 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59540/0 (socket says 172.21.15.90:59540) 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:52988/0 (socket says 172.21.15.90:52988) 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302affd700 1 -- 172.21.15.90:0/2115169587 learned_addr learned my addr 172.21.15.90:0/2115169587 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302b7fe700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 0x7f302c19de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 -- 172.21.15.90:0/2115169587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 msgr2=0x7f302c19de80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 0x7f302c19de80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 -- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 msgr2=0x7f302c1a35e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302a7fc700 1 -- 172.21.15.90:0/2115169587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f301c009520 con 0x7f302c104f70 2024-04-06T14:11:35.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302b7fe700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 0x7f302c19de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.877+0000 7f302affd700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.878+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f3020005fd0 tx=0x7f302000eca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.878+0000 7f3013fff700 1 -- 172.21.15.90:0/2115169587 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f302000c9c0 con 0x7f302c104f70 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.878+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f302c1aa440 con 0x7f302c104f70 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.878+0000 7f3013fff700 1 -- 172.21.15.90:0/2115169587 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3020010430 con 0x7f302c104f70 2024-04-06T14:11:35.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.878+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f302c1aa950 con 0x7f302c104f70 2024-04-06T14:11:35.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.880+0000 7f3013fff700 1 -- 172.21.15.90:0/2115169587 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3020018680 con 0x7f302c104f70 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.881+0000 7f3013fff700 1 -- 172.21.15.90:0/2115169587 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f30200188a0 con 0x7f302c104f70 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.881+0000 7f3013fff700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 0x7f30140a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.881+0000 7f302affd700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 0x7f30140a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.881+0000 7f3013fff700 1 -- 172.21.15.90:0/2115169587 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f3020014070 con 0x7f302c104f70 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f302affd700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 0x7f30140a3750 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f301c009fd0 tx=0x7f301c019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 Processor -- start 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 -- start start 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b18a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff042ff990 con 0x55ff043a7780 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff042fd350 con 0x55ff043a80a0 2024-04-06T14:11:35.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f303d42ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff042f8c40 con 0x55ff043b1de0 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59552/0 (socket says 172.21.15.90:59552) 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f302b7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.882+0000 7f302affd700 1 -- 172.21.15.90:0/2600159937 learned_addr learned my addr 172.21.15.90:0/2600159937 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 -- 172.21.15.90:0/2600159937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 msgr2=0x55ff043b41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043b41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 -- 172.21.15.90:0/2600159937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 msgr2=0x55ff043b18a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b18a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 -- 172.21.15.90:0/2600159937 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff0417a3b0 con 0x55ff043a7780 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.883+0000 7f302affd700 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f301c009230 tx=0x7f301c01f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b5295e22092362d8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.884+0000 7f30117fa700 1 -- 172.21.15.90:0/2600159937 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c00b8e0 con 0x55ff043a7780 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.884+0000 7f30117fa700 1 -- 172.21.15.90:0/2600159937 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f301c01d040 con 0x55ff043a7780 2024-04-06T14:11:35.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.884+0000 7f30117fa700 1 -- 172.21.15.90:0/2600159937 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c02d8e0 con 0x55ff043a7780 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 -- 172.21.15.90:0/2600159937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 msgr2=0x55ff043a7b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 secure :-1 s=READY pgs=589 cs=0 l=1 rev1=1 crypto rx=0x7f301c009230 tx=0x7f301c01f000 comp rx=0 tx=0).stop 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 -- 172.21.15.90:0/2600159937 shutdown_connections 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043b41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043a7b60 unknown :-1 s=CLOSED pgs=589 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2600159937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b18a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.885+0000 7f303d42ab80 1 -- 172.21.15.90:0/2600159937 >> 172.21.15.90:0/2600159937 conn(0x55ff043066a0 msgr2=0x55ff043a2a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- 172.21.15.90:0/2600159937 shutdown_connections 2024-04-06T14:11:35.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- 172.21.15.90:0/2600159937 wait complete. 2024-04-06T14:11:35.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- start start 2024-04-06T14:11:35.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 Processor -- start 2024-04-06T14:11:35.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b1870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043aebb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 0x55ff043af2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff04294d80 con 0x55ff043a7780 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff04294a20 con 0x55ff043b1de0 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff0417c6a0 con 0x55ff043a80a0 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.888+0000 7f303d42ab80 10 client.0 _ll_register_callbacks cb 0x55ff0429e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 0x55ff043af2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 0x55ff043af2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60174/0 (socket says 172.21.15.90:60174) 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302b7fe700 1 -- 172.21.15.90:0/857810360 learned_addr learned my addr 172.21.15.90:0/857810360 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b1870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 -- 172.21.15.90:0/857810360 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 msgr2=0x55ff043aebb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043aebb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 -- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 msgr2=0x55ff043af2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 0x55ff043af2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.889+0000 7f302affd700 1 -- 172.21.15.90:0/857810360 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f302c10d360 con 0x55ff043a7780 2024-04-06T14:11:35.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f302affd700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b1870 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f301c009810 tx=0x7f301c005e40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=470ca9dc0495fd12 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:35.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c00b8e0 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f301c00ba40 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ff042b9380 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f301c0054c0 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 10 client.16431 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ff04126b60 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ff042b9380 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f301c046070 con 0x55ff043a7780 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 10 client.16431 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 10 client.16431 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 20 client.16431 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.890+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff043d2670 0x55ff043d4a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.891+0000 7f303d42ab80 4 client.16431 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:35.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.891+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ff042f1700 con 0x55ff043d2670 2024-04-06T14:11:35.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.892+0000 7f302a7fc700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff043d2670 0x55ff043d4a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.892+0000 7f302a7fc700 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff043d2670 0x55ff043d4a60 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=429afd709081f12 server_cookie=2777ee014775fb12 in_seq=0 out_seq=0 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.892+0000 7f300eff5700 10 client.16431 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.893+0000 7f300eff5700 1 -- 172.21.15.90:0/857810360 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ff042f1700 con 0x55ff043d2670 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.893+0000 7f300eff5700 10 client.16431 handle_command_reply: tid=0 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.893+0000 7f303d42ab80 1 client.16431 shutdown 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.893+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 msgr2=0x55ff043b1870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.893+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b1870 secure :-1 s=READY pgs=590 cs=0 l=1 rev1=1 crypto rx=0x7f301c009810 tx=0x7f301c005e40 comp rx=0 tx=0).stop 2024-04-06T14:11:35.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 shutdown_connections 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff043d2670 0x55ff043d4a60 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043b1de0 0x55ff043af2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b1870 unknown :-1 s=CLOSED pgs=590 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 --2- 172.21.15.90:0/857810360 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043aebb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f300eff5700 0 client.16431 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 >> 172.21.15.90:0/857810360 conn(0x55ff043066a0 msgr2=0x55ff043a1c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 shutdown_connections 2024-04-06T14:11:35.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.894+0000 7f303d42ab80 1 -- 172.21.15.90:0/857810360 wait complete. 2024-04-06T14:11:35.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.895+0000 7f303d42ab80 20 client.16431 trim_cache size 0 max 16384 2024-04-06T14:11:35.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.910+0000 7f303d42ab80 1 Processor -- start 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 -- start start 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b8a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043bd1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff043a8a50 con 0x55ff043a7780 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff043afba0 con 0x55ff043a80a0 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f303d42ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff043b8fd0 con 0x55ff043b1de0 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59566/0 (socket says 172.21.15.90:59566) 2024-04-06T14:11:35.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 -- 172.21.15.90:0/169259476 learned_addr learned my addr 172.21.15.90:0/169259476 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302a7fc700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b8a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302b7fe700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043bd1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 -- 172.21.15.90:0/169259476 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 msgr2=0x55ff043bd1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043bd1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 -- 172.21.15.90:0/169259476 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 msgr2=0x55ff043b8a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b8a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302affd700 1 -- 172.21.15.90:0/169259476 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff042beea0 con 0x55ff043a7780 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.911+0000 7f302a7fc700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b8a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.912+0000 7f302affd700 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f301c02d920 tx=0x7f301c02db40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d4e54a00df2f79c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.912+0000 7f3011ffb700 1 -- 172.21.15.90:0/169259476 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c02d730 con 0x55ff043a7780 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.912+0000 7f3011ffb700 1 -- 172.21.15.90:0/169259476 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f301c0054c0 con 0x55ff043a7780 2024-04-06T14:11:35.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.912+0000 7f3011ffb700 1 -- 172.21.15.90:0/169259476 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c0368e0 con 0x55ff043a7780 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- 172.21.15.90:0/169259476 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 msgr2=0x55ff043b41d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 secure :-1 s=READY pgs=591 cs=0 l=1 rev1=1 crypto rx=0x7f301c02d920 tx=0x7f301c02db40 comp rx=0 tx=0).stop 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- 172.21.15.90:0/169259476 shutdown_connections 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043b1de0 0x55ff043bd1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043a7780 0x55ff043b41d0 unknown :-1 s=CLOSED pgs=591 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- 172.21.15.90:0/169259476 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a80a0 0x55ff043b8a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- 172.21.15.90:0/169259476 >> 172.21.15.90:0/169259476 conn(0x55ff043066a0 msgr2=0x55ff043d93c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- 172.21.15.90:0/169259476 shutdown_connections 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- 172.21.15.90:0/169259476 wait complete. 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 -- start start 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 Processor -- start 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043b6fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.913+0000 7f303d42ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043b1de0 0x55ff04425470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f303d42ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff043b7870 con 0x55ff043b1de0 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f303d42ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff044259b0 con 0x55ff043a7780 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f303d42ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff04425af0 con 0x55ff043a80a0 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f303d42ab80 10 client.0 _ll_register_callbacks cb 0x55ff0429e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043b1de0 0x55ff04425470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60196/0 (socket says 172.21.15.90:60196) 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 -- 172.21.15.90:0/2307843364 learned_addr learned my addr 172.21.15.90:0/2307843364 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043b6fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 -- 172.21.15.90:0/2307843364 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 msgr2=0x55ff043b6fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043b6fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 -- 172.21.15.90:0/2307843364 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043b1de0 msgr2=0x55ff04425470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043b1de0 0x55ff04425470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 -- 172.21.15.90:0/2307843364 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f302c1aa950 con 0x55ff043a7780 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f302affd700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f301c00c010 tx=0x7f301c00b870 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f773ec06d94d9efc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f300eff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:35.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f301c0365f0 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f301c02dc10 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.914+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f301c01d040 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f301c036750 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 10 client.26150 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ff04126b60 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f301c060020 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f301c00f790 con 0x55ff043a7780 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 10 client.26150 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 10 client.26150 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 20 client.26150 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff04426d50 0x55ff04429140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:35.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 4 client.26150 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ff04147cf0 con 0x55ff04426d50 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.915+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff04426d50 0x55ff04429140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f300eff5700 10 client.26150 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f302a7fc700 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff04426d50 0x55ff04429140 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f9400b83d26aed1 server_cookie=c9d2acefa567eb3d in_seq=0 out_seq=0 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f300eff5700 1 -- 172.21.15.90:0/2307843364 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ff04147cf0 con 0x55ff04426d50 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f300eff5700 10 client.26150 handle_command_reply: tid=0 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 client.26150 shutdown 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 msgr2=0x55ff043b6a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f301c00c010 tx=0x7f301c00b870 comp rx=0 tx=0).stop 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 shutdown_connections 2024-04-06T14:11:35.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ff04426d50 0x55ff04429140 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ff043b1de0 0x55ff04425470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ff043a7780 0x55ff043b6a70 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 --2- 172.21.15.90:0/2307843364 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ff043a80a0 0x55ff043b6fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f300eff5700 0 client.26150 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 >> 172.21.15.90:0/2307843364 conn(0x55ff043066a0 msgr2=0x55ff043bd1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 shutdown_connections 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 1 -- 172.21.15.90:0/2307843364 wait complete. 2024-04-06T14:11:35.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.916+0000 7f303d42ab80 20 client.26150 trim_cache size 0 max 16384 2024-04-06T14:11:35.924 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:35.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 msgr2=0x7f30140a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 0x7f30140a3750 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f301c009fd0 tx=0x7f301c019040 comp rx=0 tx=0).stop 2024-04-06T14:11:35.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 msgr2=0x7f302c1a3b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:35.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f3020005fd0 tx=0x7f302000eca0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 shutdown_connections 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f30140a1290 0x7f30140a3750 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f302c19d9d0 0x7f302c19de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f302c0ffe70 0x7f302c1a35e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 --2- 172.21.15.90:0/2115169587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f302c104f70 0x7f302c1a3b20 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.918+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 >> 172.21.15.90:0/2115169587 conn(0x7f302c076d50 msgr2=0x7f302c068120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.919+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 shutdown_connections 2024-04-06T14:11:35.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:35.919+0000 7f3031276700 1 -- 172.21.15.90:0/2115169587 wait complete. 2024-04-06T14:11:35.933 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:35.933 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:11:35.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1734590897' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:35.934 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:35 smithi090 ceph-mon[43420]: pgmap v189: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 16 MiB/s wr, 1.03k op/s 2024-04-06T14:11:36.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1734590897' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:36.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:35 smithi134 ceph-mon[47549]: pgmap v189: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 16 MiB/s wr, 1.03k op/s 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3019411088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c441005a0 msgr2=0x7f4c44100980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3019411088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c441005a0 0x7f4c44100980 secure :-1 s=READY pgs=592 cs=0 l=1 rev1=1 crypto rx=0x7f4c38009990 tx=0x7f4c38009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3019411088 shutdown_connections 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3019411088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 0x7f4c4410cd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3019411088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c441005a0 0x7f4c44100980 unknown :-1 s=CLOSED pgs=592 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3019411088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c44100f50 0x7f4c44104a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3019411088 >> 172.21.15.90:0/3019411088 conn(0x7f4c440761b0 msgr2=0x7f4c440fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3019411088 shutdown_connections 2024-04-06T14:11:36.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.155+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3019411088 wait complete. 2024-04-06T14:11:36.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 Processor -- start 2024-04-06T14:11:36.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 -- start start 2024-04-06T14:11:36.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 0x7f4c441981e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 0x7f4c44198720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 0x7f4c441a3260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c441a3970 con 0x7f4c44198cb0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c441a3ae0 con 0x7f4c441051a0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c4af4e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4c44195fd0 con 0x7f4c441005a0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c494eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 0x7f4c441a3260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c494eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 0x7f4c441a3260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59584/0 (socket says 172.21.15.90:59584) 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c494eb700 1 -- 172.21.15.90:0/3550683807 learned_addr learned my addr 172.21.15.90:0/3550683807 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 0x7f4c441981e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.156+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 0x7f4c44198720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 -- 172.21.15.90:0/3550683807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 msgr2=0x7f4c44198720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 0x7f4c44198720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 -- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 msgr2=0x7f4c441a3260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 0x7f4c441a3260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 -- 172.21.15.90:0/3550683807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4c30009520 con 0x7f4c441005a0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 0x7f4c441981e0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f4c380059d0 tx=0x7f4c38004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c41ffb700 1 -- 172.21.15.90:0/3550683807 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3801d070 con 0x7f4c441005a0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4c380095f0 con 0x7f4c441005a0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c41ffb700 1 -- 172.21.15.90:0/3550683807 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4c38022470 con 0x7f4c441005a0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c41ffb700 1 -- 172.21.15.90:0/3550683807 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c38017650 con 0x7f4c441005a0 2024-04-06T14:11:36.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.157+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c44196540 con 0x7f4c441005a0 2024-04-06T14:11:36.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.161+0000 7f4c41ffb700 1 -- 172.21.15.90:0/3550683807 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4c38022870 con 0x7f4c441005a0 2024-04-06T14:11:36.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.161+0000 7f4c41ffb700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 0x7f4c340a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.161+0000 7f4c41ffb700 1 -- 172.21.15.90:0/3550683807 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4c380cdb30 con 0x7f4c441005a0 2024-04-06T14:11:36.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.162+0000 7f4c57102b80 1 Processor -- start 2024-04-06T14:11:36.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.163+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 0x7f4c340a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.163+0000 7f4c57102b80 1 -- start start 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c8b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c8e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6bd9990 con 0x55fad6c81780 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 0x7f4c340a37d0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f4c440ffa70 tx=0x7f4c30009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6bd7350 con 0x55fad6c8bde0 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c57102b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6bd2c40 con 0x55fad6c820a0 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c43fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c8b8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59596/0 (socket says 172.21.15.90:59596) 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 -- 172.21.15.90:0/1099527041 learned_addr learned my addr 172.21.15.90:0/1099527041 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c8e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 -- 172.21.15.90:0/1099527041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 msgr2=0x55fad6c8b8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c8b8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 -- 172.21.15.90:0/1099527041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 msgr2=0x55fad6c8e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c8e1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 -- 172.21.15.90:0/1099527041 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fad6a543b0 con 0x55fad6c81780 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c8e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:36.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.164+0000 7f4c48cea700 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f4c380a2ba0 tx=0x7f4c3809a870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9d6737e9d17554a4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.165+0000 7f4c2effd700 1 -- 172.21.15.90:0/1099527041 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3809ace0 con 0x55fad6c81780 2024-04-06T14:11:36.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.165+0000 7f4c2effd700 1 -- 172.21.15.90:0/1099527041 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4c3809eb10 con 0x55fad6c81780 2024-04-06T14:11:36.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.165+0000 7f4c2effd700 1 -- 172.21.15.90:0/1099527041 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3809a1d0 con 0x55fad6c81780 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.168+0000 7f4c57102b80 1 -- 172.21.15.90:0/1099527041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 msgr2=0x55fad6c81b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.168+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 secure :-1 s=READY pgs=593 cs=0 l=1 rev1=1 crypto rx=0x7f4c380a2ba0 tx=0x7f4c3809a870 comp rx=0 tx=0).stop 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.169+0000 7f4c57102b80 1 -- 172.21.15.90:0/1099527041 shutdown_connections 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.169+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c8e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.169+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c81b60 unknown :-1 s=CLOSED pgs=593 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.169+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1099527041 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c8b8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.169+0000 7f4c57102b80 1 -- 172.21.15.90:0/1099527041 >> 172.21.15.90:0/1099527041 conn(0x55fad6be06a0 msgr2=0x55fad6c7ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.169 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.170+0000 7f4c57102b80 1 -- 172.21.15.90:0/1099527041 shutdown_connections 2024-04-06T14:11:36.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.181+0000 7f4c57102b80 1 -- 172.21.15.90:0/1099527041 wait complete. 2024-04-06T14:11:36.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.181+0000 7f4c57102b80 1 -- start start 2024-04-06T14:11:36.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.181+0000 7f4c57102b80 1 Processor -- start 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c8b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c88ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c89180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6b6ed80 con 0x55fad6c8bde0 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6b6ea20 con 0x55fad6c81780 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6a566a0 con 0x55fad6c820a0 2024-04-06T14:11:36.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.183+0000 7f4c57102b80 10 client.0 _ll_register_callbacks cb 0x55fad6b78470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.184+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c8b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.184+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c8b7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60228/0 (socket says 172.21.15.90:60228) 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.184+0000 7f4c48cea700 1 -- 172.21.15.90:0/1691614225 learned_addr learned my addr 172.21.15.90:0/1691614225 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.184+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c89180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 -- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 msgr2=0x55fad6c88ae0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c88ae0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 -- 172.21.15.90:0/1691614225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 msgr2=0x55fad6c8b7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c8b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 -- 172.21.15.90:0/1691614225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c3c009a40 con 0x55fad6c8bde0 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c494eb700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c89180 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f4c3c009550 tx=0x7f4c3c007c70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af8285ec06453b70 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.185+0000 7f4c23fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.190+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3c0139b0 con 0x55fad6c8bde0 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.190+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4c3c013b10 con 0x55fad6c8bde0 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.190+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55fad6b93380 con 0x55fad6c8bde0 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.191+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4c3c013cd0 con 0x55fad6c8bde0 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.191+0000 7f4c57102b80 10 client.16449 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:36.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.191+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f4c44196540 con 0x55fad6c8bde0 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.191+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55fad6b93380 con 0x55fad6c8bde0 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.191+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4c3c04b420 con 0x55fad6c8bde0 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 10 client.16449 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 10 client.16449 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 20 client.16449 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6ca2810 0x55fad6ca4c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 4 client.16449 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:36.192 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.192+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55fad6bcb700 con 0x55fad6ca2810 2024-04-06T14:11:36.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.199+0000 7f4c48cea700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6ca2810 0x55fad6ca4c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.234+0000 7f4c23fff700 10 client.16449 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:36.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.234+0000 7f4c48cea700 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6ca2810 0x55fad6ca4c30 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=15b6a5fdbdbd4d59 server_cookie=a0f3434501c0733b in_seq=0 out_seq=0 2024-04-06T14:11:36.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.236+0000 7f4c23fff700 1 -- 172.21.15.90:0/1691614225 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55fad6bcb700 con 0x55fad6ca2810 2024-04-06T14:11:36.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.236+0000 7f4c23fff700 10 client.16449 handle_command_reply: tid=0 2024-04-06T14:11:36.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.236+0000 7f4c57102b80 1 client.16449 shutdown 2024-04-06T14:11:36.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 msgr2=0x55fad6c89180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c89180 secure :-1 s=READY pgs=594 cs=0 l=1 rev1=1 crypto rx=0x7f4c3c009550 tx=0x7f4c3c007c70 comp rx=0 tx=0).stop 2024-04-06T14:11:36.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 shutdown_connections 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6ca2810 0x55fad6ca4c30 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c89180 unknown :-1 s=CLOSED pgs=594 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c8b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 --2- 172.21.15.90:0/1691614225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c88ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 >> 172.21.15.90:0/1691614225 conn(0x55fad6be06a0 msgr2=0x55fad6c7bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.239+0000 7f4c23fff700 0 client.16449 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:36.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.240+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 shutdown_connections 2024-04-06T14:11:36.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.240+0000 7f4c57102b80 1 -- 172.21.15.90:0/1691614225 wait complete. 2024-04-06T14:11:36.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.240+0000 7f4c57102b80 20 client.16449 trim_cache size 0 max 16384 2024-04-06T14:11:36.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1734590897' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:11:36.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:35 smithi105 ceph-mon[46969]: pgmap v189: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 16 MiB/s wr, 1.03k op/s 2024-04-06T14:11:36.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 Processor -- start 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 -- start start 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c873f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c87930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6c82a10 con 0x55fad6c8bde0 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6c87e70 con 0x55fad6c81780 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c57102b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6c7ddc0 con 0x55fad6c820a0 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c494eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c494eb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59606/0 (socket says 172.21.15.90:59606) 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c494eb700 1 -- 172.21.15.90:0/3490675629 learned_addr learned my addr 172.21.15.90:0/3490675629 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c873f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.260+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c87930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 -- 172.21.15.90:0/3490675629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 msgr2=0x55fad6c87930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c87930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 -- 172.21.15.90:0/3490675629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 msgr2=0x55fad6c873f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c873f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 -- 172.21.15.90:0/3490675629 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fad6a00b60 con 0x55fad6c8bde0 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c494eb700 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f4c3c007f80 tx=0x7f4c3c009390 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ce14f6e0cc67e3eb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c2f7fe700 1 -- 172.21.15.90:0/3490675629 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3c009220 con 0x55fad6c8bde0 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c2f7fe700 1 -- 172.21.15.90:0/3490675629 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4c3c00f040 con 0x55fad6c8bde0 2024-04-06T14:11:36.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.261+0000 7f4c2f7fe700 1 -- 172.21.15.90:0/3490675629 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3c013aa0 con 0x55fad6c8bde0 2024-04-06T14:11:36.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 -- 172.21.15.90:0/3490675629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 msgr2=0x55fad6c7c3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 secure :-1 s=READY pgs=595 cs=0 l=1 rev1=1 crypto rx=0x7f4c3c007f80 tx=0x7f4c3c009390 comp rx=0 tx=0).stop 2024-04-06T14:11:36.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 -- 172.21.15.90:0/3490675629 shutdown_connections 2024-04-06T14:11:36.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c8bde0 0x55fad6c7c3a0 unknown :-1 s=CLOSED pgs=595 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c81780 0x55fad6c873f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3490675629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c87930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 -- 172.21.15.90:0/3490675629 >> 172.21.15.90:0/3490675629 conn(0x55fad6be06a0 msgr2=0x55fad6c83e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 -- 172.21.15.90:0/3490675629 shutdown_connections 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.262+0000 7f4c57102b80 1 -- 172.21.15.90:0/3490675629 wait complete. 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.263+0000 7f4c57102b80 1 -- start start 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.263+0000 7f4c57102b80 1 Processor -- start 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c98f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c99670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6c99d50 con 0x55fad6c81780 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6cf7830 con 0x55fad6c8bde0 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad6cf7970 con 0x55fad6c820a0 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c57102b80 10 client.0 _ll_register_callbacks cb 0x55fad6b78470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:36.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59620/0 (socket says 172.21.15.90:59620) 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 -- 172.21.15.90:0/3855298676 learned_addr learned my addr 172.21.15.90:0/3855298676 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c98f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c494eb700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c99670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 -- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 msgr2=0x55fad6c98f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c98f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 -- 172.21.15.90:0/3855298676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 msgr2=0x55fad6c99670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c99670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c48cea700 1 -- 172.21.15.90:0/3855298676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4c44196540 con 0x55fad6c81780 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.264+0000 7f4c494eb700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c99670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c48cea700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f4c38097750 tx=0x7f4c380047a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=159aa365605083a7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c23fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4c3809ace0 con 0x55fad6c81780 2024-04-06T14:11:36.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f4c3809a1d0 con 0x55fad6c81780 2024-04-06T14:11:36.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4c3809eb10 con 0x55fad6c81780 2024-04-06T14:11:36.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.265+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4c3809a390 con 0x55fad6c81780 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.266+0000 7f4c57102b80 10 client.16455 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55fad6b98ea0 con 0x55fad6c81780 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4c3809eb10 con 0x55fad6c81780 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4c380a2c40 con 0x55fad6c81780 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 10 client.16455 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 10 client.16455 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 20 client.16455 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:36.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6cf8bb0 0x55fad6cfafd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 4 client.16455 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55fad6a21cf0 con 0x55fad6cf8bb0 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.267+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6cf8bb0 0x55fad6cfafd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.268+0000 7f4c23fff700 10 client.16455 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.268+0000 7f4c43fff700 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6cf8bb0 0x55fad6cfafd0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=cd54306e6f21ee8c server_cookie=12357c7a2228bd98 in_seq=0 out_seq=0 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.268+0000 7f4c23fff700 1 -- 172.21.15.90:0/3855298676 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55fad6a21cf0 con 0x55fad6cf8bb0 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.268+0000 7f4c23fff700 10 client.16455 handle_command_reply: tid=0 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.268+0000 7f4c57102b80 1 client.16455 shutdown 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.269+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 msgr2=0x55fad6c9b140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.269+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 secure :-1 s=READY pgs=596 cs=0 l=1 rev1=1 crypto rx=0x7f4c38097750 tx=0x7f4c380047a0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 shutdown_connections 2024-04-06T14:11:36.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55fad6cf8bb0 0x55fad6cfafd0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fad6c8bde0 0x55fad6c99670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fad6c81780 0x55fad6c9b140 unknown :-1 s=CLOSED pgs=596 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 --2- 172.21.15.90:0/3855298676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fad6c820a0 0x55fad6c98f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 >> 172.21.15.90:0/3855298676 conn(0x55fad6be06a0 msgr2=0x55fad6c872c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c23fff700 0 client.16455 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 shutdown_connections 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 1 -- 172.21.15.90:0/3855298676 wait complete. 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.273+0000 7f4c57102b80 20 client.16455 trim_cache size 0 max 16384 2024-04-06T14:11:36.273 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:36.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 msgr2=0x7f4c340a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 0x7f4c340a37d0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f4c440ffa70 tx=0x7f4c30009260 comp rx=0 tx=0).stop 2024-04-06T14:11:36.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 msgr2=0x7f4c441981e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 0x7f4c441981e0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f4c380059d0 tx=0x7f4c38004730 comp rx=0 tx=0).stop 2024-04-06T14:11:36.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 shutdown_connections 2024-04-06T14:11:36.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4c340a1310 0x7f4c340a37d0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4c44198cb0 0x7f4c441a3260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4c441005a0 0x7f4c441981e0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 --2- 172.21.15.90:0/3550683807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4c441051a0 0x7f4c44198720 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.274+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 >> 172.21.15.90:0/3550683807 conn(0x7f4c440761b0 msgr2=0x7f4c44103400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.277+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 shutdown_connections 2024-04-06T14:11:36.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.277+0000 7f4c4af4e700 1 -- 172.21.15.90:0/3550683807 wait complete. 2024-04-06T14:11:36.291 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:11:36.291 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 -- 172.21.15.90:0/2743450302 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec105d00 msgr2=0x7f8cec1060e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/2743450302 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec105d00 0x7f8cec1060e0 secure :-1 s=READY pgs=597 cs=0 l=1 rev1=1 crypto rx=0x7f8ce8009990 tx=0x7f8ce8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 -- 172.21.15.90:0/2743450302 shutdown_connections 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/2743450302 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec10dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/2743450302 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec105d00 0x7f8cec1060e0 unknown :-1 s=CLOSED pgs=597 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/2743450302 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec0ff220 0x7f8cec0ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 -- 172.21.15.90:0/2743450302 >> 172.21.15.90:0/2743450302 conn(0x7f8cec0761b0 msgr2=0x7f8cec0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 -- 172.21.15.90:0/2743450302 shutdown_connections 2024-04-06T14:11:36.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.462+0000 7f8cf3f17700 1 -- 172.21.15.90:0/2743450302 wait complete. 2024-04-06T14:11:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 Processor -- start 2024-04-06T14:11:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 -- start start 2024-04-06T14:11:36.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec197890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 0x7f8cec19bc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cec19c270 con 0x7f8cec0ff220 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cec19c3e0 con 0x7f8cec0ffbe0 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf3f17700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8cec19c5a0 con 0x7f8cec105d00 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59642/0 (socket says 172.21.15.90:59642) 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.463+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/1103123129 learned_addr learned my addr 172.21.15.90:0/1103123129 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 0x7f8cec19bc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec197890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/1103123129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 msgr2=0x7f8cec19bc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 0x7f8cec19bc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/1103123129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 msgr2=0x7f8cec197890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec197890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/1103123129 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ce80095f0 con 0x7f8cec0ff220 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 0x7f8cec19bc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec197890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f8ce80095c0 tx=0x7f8ce800f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8ce2ffd700 1 -- 172.21.15.90:0/1103123129 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce801d070 con 0x7f8cec0ff220 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8cec19c820 con 0x7f8cec0ff220 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8ce2ffd700 1 -- 172.21.15.90:0/1103123129 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ce80054f0 con 0x7f8cec0ff220 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8cec1a40c0 con 0x7f8cec0ff220 2024-04-06T14:11:36.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.464+0000 7f8ce2ffd700 1 -- 172.21.15.90:0/1103123129 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce8004bd0 con 0x7f8cec0ff220 2024-04-06T14:11:36.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.467+0000 7f8ce2ffd700 1 -- 172.21.15.90:0/1103123129 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8ce800fb80 con 0x7f8cec0ff220 2024-04-06T14:11:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.468+0000 7f8d000cbb80 1 Processor -- start 2024-04-06T14:11:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.468+0000 7f8ce2ffd700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 0x7f8cd80a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.469+0000 7f8ce2ffd700 1 -- 172.21.15.90:0/1103123129 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8ce80cd940 con 0x7f8cec0ff220 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.469+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 0x7f8cd80a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.469+0000 7f8d000cbb80 1 -- start start 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb892fc0 0x55d4bb8933a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb884c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb7dc990 con 0x55d4bb892fc0 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb7da350 con 0x55d4bb885140 2024-04-06T14:11:36.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb7d5c40 con 0x55d4bb884780 2024-04-06T14:11:36.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60274/0 (socket says 172.21.15.90:60274) 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 -- 172.21.15.90:0/1954276410 learned_addr learned my addr 172.21.15.90:0/1954276410 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 -- 172.21.15.90:0/1954276410 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 msgr2=0x55d4bb884c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb884c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 -- 172.21.15.90:0/1954276410 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb892fc0 msgr2=0x55d4bb8933a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.470+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb892fc0 0x55d4bb8933a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cf24b4700 1 -- 172.21.15.90:0/1954276410 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d4bb6573b0 con 0x55d4bb885140 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f8ce4019bd0 tx=0x7f8ce401c000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=34d8b6c72c2d6407 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 0x7f8cd80a3750 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f8ce4019480 tx=0x7f8ce4007bf0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cd7fff700 1 -- 172.21.15.90:0/1954276410 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce401bc40 con 0x55d4bb885140 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cd7fff700 1 -- 172.21.15.90:0/1954276410 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ce400f040 con 0x55d4bb885140 2024-04-06T14:11:36.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.471+0000 7f8cd7fff700 1 -- 172.21.15.90:0/1954276410 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce401a640 con 0x55d4bb885140 2024-04-06T14:11:36.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 -- 172.21.15.90:0/1954276410 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 msgr2=0x55d4bb899d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f8ce4019bd0 tx=0x7f8ce401c000 comp rx=0 tx=0).stop 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 -- 172.21.15.90:0/1954276410 shutdown_connections 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb885140 0x55d4bb899d00 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb892fc0 0x55d4bb8933a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/1954276410 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb884c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.472+0000 7f8d000cbb80 1 -- 172.21.15.90:0/1954276410 >> 172.21.15.90:0/1954276410 conn(0x55d4bb7e36a0 msgr2=0x55d4bb87f720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.473+0000 7f8d000cbb80 1 -- 172.21.15.90:0/1954276410 shutdown_connections 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.473+0000 7f8d000cbb80 1 -- 172.21.15.90:0/1954276410 wait complete. 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.473+0000 7f8d000cbb80 1 -- start start 2024-04-06T14:11:36.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.473+0000 7f8d000cbb80 1 Processor -- start 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb897590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb897ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb771d80 con 0x55d4bb884780 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb771a20 con 0x55d4bb892fc0 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb6596a0 con 0x55d4bb885140 2024-04-06T14:11:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8d000cbb80 10 client.0 _ll_register_callbacks cb 0x55d4bb77b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59656/0 (socket says 172.21.15.90:59656) 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2508224304 learned_addr learned my addr 172.21.15.90:0/2508224304 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb897ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2508224304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 msgr2=0x55d4bb897590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb897590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb897590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 msgr2=0x55d4bb897ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb897ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2508224304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ce4019dc0 con 0x55d4bb884780 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb897590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.474+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb897ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f8ce8023000 tx=0x7f8ce809a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=558f619e1a677008 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce809abd0 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8ce809ea00 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d4bb796380 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8ce80a2fc0 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 10 client.16464 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d4bb603b60 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d4bb796380 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8ce80969c0 con 0x55d4bb884780 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 10 client.16464 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 10 client.16464 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 20 client.16464 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb8a6f40 0x55d4bb8a9360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 4 client.16464 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.475+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d4bb7ce700 con 0x55d4bb8a6f40 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.476+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb8a6f40 0x55d4bb8a9360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.476+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb8a6f40 0x55d4bb8a9360 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=95d6d1713d48048d server_cookie=7d59495dec5c17fe in_seq=0 out_seq=0 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.476+0000 7f8cd57fa700 10 client.16464 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2508224304 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d4bb7ce700 con 0x55d4bb8a6f40 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8cd57fa700 10 client.16464 handle_command_reply: tid=0 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 client.16464 shutdown 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 msgr2=0x55d4bb897050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 secure :-1 s=READY pgs=599 cs=0 l=1 rev1=1 crypto rx=0x7f8ce8023000 tx=0x7f8ce809a660 comp rx=0 tx=0).stop 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 shutdown_connections 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb897ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb897050 unknown :-1 s=CLOSED pgs=599 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb8a6f40 0x55d4bb8a9360 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2508224304 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb897590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8cd57fa700 0 client.16464 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 >> 172.21.15.90:0/2508224304 conn(0x55d4bb7e36a0 msgr2=0x55d4bb8922c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 shutdown_connections 2024-04-06T14:11:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.477+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2508224304 wait complete. 2024-04-06T14:11:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.479+0000 7f8d000cbb80 20 client.16464 trim_cache size 0 max 16384 2024-04-06T14:11:36.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 Processor -- start 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 -- start start 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb8933a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 0x55d4bb894c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb890130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb898600 con 0x55d4bb885140 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb880710 con 0x55d4bb892fc0 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb8811d0 con 0x55d4bb884780 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8cf14b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 0x55d4bb894c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb8933a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8cf14b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 0x55d4bb894c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59660/0 (socket says 172.21.15.90:59660) 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8cf14b2700 1 -- 172.21.15.90:0/3973107440 learned_addr learned my addr 172.21.15.90:0/3973107440 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.496+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb890130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/3973107440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 msgr2=0x55d4bb890130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb890130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/3973107440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 msgr2=0x55d4bb894c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 0x55d4bb894c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/3973107440 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d4bb79bea0 con 0x55d4bb884780 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb8933a0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8ce8004820 tx=0x7f8ce800ff50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4da2774f14054011 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8ce0ff9700 1 -- 172.21.15.90:0/3973107440 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce80a2df0 con 0x55d4bb884780 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8ce0ff9700 1 -- 172.21.15.90:0/3973107440 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ce8096f10 con 0x55d4bb884780 2024-04-06T14:11:36.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.497+0000 7f8ce0ff9700 1 -- 172.21.15.90:0/3973107440 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce809a380 con 0x55d4bb884780 2024-04-06T14:11:36.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- 172.21.15.90:0/3973107440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 msgr2=0x55d4bb8933a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb8933a0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8ce8004820 tx=0x7f8ce800ff50 comp rx=0 tx=0).stop 2024-04-06T14:11:36.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- 172.21.15.90:0/3973107440 shutdown_connections 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb890130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb884780 0x55d4bb8933a0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/3973107440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb885140 0x55d4bb894c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- 172.21.15.90:0/3973107440 >> 172.21.15.90:0/3973107440 conn(0x55d4bb7e36a0 msgr2=0x55d4bb897950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- 172.21.15.90:0/3973107440 shutdown_connections 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- 172.21.15.90:0/3973107440 wait complete. 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 -- start start 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.498+0000 7f8d000cbb80 1 Processor -- start 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb89c390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb89c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb89cf90 con 0x55d4bb884780 2024-04-06T14:11:36.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb8fa2b0 con 0x55d4bb892fc0 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d4bb885b70 con 0x55d4bb885140 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8d000cbb80 10 client.0 _ll_register_callbacks cb 0x55d4bb77b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf24b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb89c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf24b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60294/0 (socket says 172.21.15.90:60294) 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf24b4700 1 -- 172.21.15.90:0/2779407262 learned_addr learned my addr 172.21.15.90:0/2779407262 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb89c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2779407262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 msgr2=0x55d4bb89c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb89c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 msgr2=0x55d4bb8f9d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf1cb3700 1 -- 172.21.15.90:0/2779407262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d4bb603b60 con 0x55d4bb884780 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.499+0000 7f8cf24b4700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:11:36.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cf1cb3700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb89c390 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x55d4bb881360 tx=0x7f8ce809a660 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f077b95ec67afb42 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ce80a2fc0 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8ce400f040 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f8ce809abd0 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8ce80a2b30 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 10 client.16470 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d4bb65bcd0 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ce400f040 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8ce80b31f0 con 0x55d4bb884780 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 10 client.16470 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 10 client.16470 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 20 client.16470 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb886e30 0x55d4bb889220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 4 client.16470 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.500+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d4bb624cf0 con 0x55d4bb886e30 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb886e30 0x55d4bb889220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8cf14b2700 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb886e30 0x55d4bb889220 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=226e8533021c405c server_cookie=c0ae5ebaece4278a in_seq=0 out_seq=0 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8cd57fa700 10 client.16470 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8cd57fa700 1 -- 172.21.15.90:0/2779407262 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d4bb624cf0 con 0x55d4bb886e30 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8cd57fa700 10 client.16470 handle_command_reply: tid=0 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.501+0000 7f8d000cbb80 1 client.16470 shutdown 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 msgr2=0x55d4bb89c390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb89c390 secure :-1 s=READY pgs=600 cs=0 l=1 rev1=1 crypto rx=0x55d4bb881360 tx=0x7f8ce809a660 comp rx=0 tx=0).stop 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 shutdown_connections 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d4bb892fc0 0x55d4bb8f9d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d4bb884780 0x55d4bb89c390 unknown :-1 s=CLOSED pgs=600 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d4bb886e30 0x55d4bb889220 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 --2- 172.21.15.90:0/2779407262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d4bb885140 0x55d4bb89c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 >> 172.21.15.90:0/2779407262 conn(0x55d4bb7e36a0 msgr2=0x55d4bb898830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8cd57fa700 0 client.16470 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 shutdown_connections 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.502+0000 7f8d000cbb80 1 -- 172.21.15.90:0/2779407262 wait complete. 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.503+0000 7f8d000cbb80 20 client.16470 trim_cache size 0 max 16384 2024-04-06T14:11:36.507 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.504+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 msgr2=0x7f8cd80a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.504+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 0x7f8cd80a3750 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f8ce4019480 tx=0x7f8ce4007bf0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.504+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 msgr2=0x7f8cec197350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.504+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 secure :-1 s=READY pgs=598 cs=0 l=1 rev1=1 crypto rx=0x7f8ce80095c0 tx=0x7f8ce800f690 comp rx=0 tx=0).stop 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 shutdown_connections 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8cd80a1290 0x7f8cd80a3750 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8cec105d00 0x7f8cec19bc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8cec0ff220 0x7f8cec197350 unknown :-1 s=CLOSED pgs=598 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 --2- 172.21.15.90:0/1103123129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8cec0ffbe0 0x7f8cec197890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:36.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.505+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 >> 172.21.15.90:0/1103123129 conn(0x7f8cec0761b0 msgr2=0x7f8cec103ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:36.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.507+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 shutdown_connections 2024-04-06T14:11:36.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:36.507+0000 7f8cf3f17700 1 -- 172.21.15.90:0/1103123129 wait complete. 2024-04-06T14:11:37.523 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:11:37.524 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:11:37.524 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 -- 172.21.15.90:0/4000308117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3238104700 msgr2=0x7f323810c280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 --2- 172.21.15.90:0/4000308117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3238104700 0x7f323810c280 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f323000b110 tx=0x7f323000aa40 comp rx=0 tx=0).stop 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 -- 172.21.15.90:0/4000308117 shutdown_connections 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 --2- 172.21.15.90:0/4000308117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3238104700 0x7f323810c280 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 --2- 172.21.15.90:0/4000308117 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32380ff5e0 0x7f32380ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 --2- 172.21.15.90:0/4000308117 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f32380fff90 0x7f3238103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.709+0000 7f323e8ae700 1 -- 172.21.15.90:0/4000308117 >> 172.21.15.90:0/4000308117 conn(0x7f32380761b0 msgr2=0x7f32380fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.711+0000 7f323e8ae700 1 -- 172.21.15.90:0/4000308117 shutdown_connections 2024-04-06T14:11:37.710 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.711+0000 7f323e8ae700 1 -- 172.21.15.90:0/4000308117 wait complete. 2024-04-06T14:11:37.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.711+0000 7f323e8ae700 1 Processor -- start 2024-04-06T14:11:37.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 -- start start 2024-04-06T14:11:37.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3238104700 0x7f3238195440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.711 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32381997e0 0x7f3238199c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f323819a260 con 0x7f3238104700 2024-04-06T14:11:37.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f323819a3d0 con 0x7f32380ff5e0 2024-04-06T14:11:37.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f323e8ae700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f323819a590 con 0x7f32381997e0 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f3237fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f3237fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60314/0 (socket says 172.21.15.90:60314) 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.712+0000 7f3237fff700 1 -- 172.21.15.90:0/2373726931 learned_addr learned my addr 172.21.15.90:0/2373726931 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f323ce4b700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32381997e0 0x7f3238199c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 -- 172.21.15.90:0/2373726931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32381997e0 msgr2=0x7f3238199c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32381997e0 0x7f3238199c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 -- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3238104700 msgr2=0x7f3238195440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.713 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3238104700 0x7f3238195440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 -- 172.21.15.90:0/2373726931 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f322c00d930 con 0x7f32380ff5e0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f3237fff700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f3228007bd0 tx=0x7f3228007f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f32357fa700 1 -- 172.21.15.90:0/2373726931 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3228010040 con 0x7f32380ff5e0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f323000ad20 con 0x7f32380ff5e0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f32357fa700 1 -- 172.21.15.90:0/2373726931 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3228015470 con 0x7f32380ff5e0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.713+0000 7f32357fa700 1 -- 172.21.15.90:0/2373726931 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322800b810 con 0x7f32380ff5e0 2024-04-06T14:11:37.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.714+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f323806a7d0 con 0x7f32380ff5e0 2024-04-06T14:11:37.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.716+0000 7f32357fa700 1 -- 172.21.15.90:0/2373726931 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f32280155e0 con 0x7f32380ff5e0 2024-04-06T14:11:37.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.717+0000 7f32357fa700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 0x7f32200b47e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.716 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.717+0000 7f32357fa700 1 -- 172.21.15.90:0/2373726931 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f32280cd200 con 0x7f32380ff5e0 2024-04-06T14:11:37.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 Processor -- start 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 -- start start 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85b8be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85c2920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c5250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab8510a10 con 0x559ab85b8800 2024-04-06T14:11:37.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab850e3d0 con 0x559ab85b9120 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f324aa62b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab8509cc0 con 0x559ab85c2e60 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.718+0000 7f3237fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85b8be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f3237fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85b8be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59688/0 (socket says 172.21.15.90:59688) 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f3237fff700 1 -- 172.21.15.90:0/2401874100 learned_addr learned my addr 172.21.15.90:0/2401874100 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85c2920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 0x7f32200b47e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 -- 172.21.15.90:0/2401874100 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 msgr2=0x559ab85c5250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c5250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 -- 172.21.15.90:0/2401874100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 msgr2=0x559ab85b8be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85b8be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.719 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 -- 172.21.15.90:0/2401874100 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559ab838b430 con 0x559ab85b9120 2024-04-06T14:11:37.722 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f32377fe700 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85c2920 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f3238196450 tx=0x7f322c016dc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9419dc0c0b1622ed server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f321e7fc700 1 -- 172.21.15.90:0/2401874100 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322c004260 con 0x559ab85b9120 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f321e7fc700 1 -- 172.21.15.90:0/2401874100 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f322c0043c0 con 0x559ab85b9120 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.719+0000 7f321e7fc700 1 -- 172.21.15.90:0/2401874100 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322c023480 con 0x559ab85b9120 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f32377fe700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 0x7f32200b47e0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f322c021000 tx=0x7f322c023000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 -- 172.21.15.90:0/2401874100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 msgr2=0x559ab85c2920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85c2920 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f3238196450 tx=0x7f322c016dc0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 -- 172.21.15.90:0/2401874100 shutdown_connections 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c5250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85b8be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2401874100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85c2920 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 -- 172.21.15.90:0/2401874100 >> 172.21.15.90:0/2401874100 conn(0x559ab8517720 msgr2=0x559ab85b3b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 -- 172.21.15.90:0/2401874100 shutdown_connections 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.720+0000 7f324aa62b80 1 -- 172.21.15.90:0/2401874100 wait complete. 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 -- start start 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 Processor -- start 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b9120 0x559ab85bfb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 0x559ab85c01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab84a5e00 con 0x559ab85c2e60 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab84a5aa0 con 0x559ab85b8800 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab838d720 con 0x559ab85b9120 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f324aa62b80 10 client.0 _ll_register_callbacks cb 0x559ab84af4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f3237fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f3237fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60324/0 (socket says 172.21.15.90:60324) 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.721+0000 7f3237fff700 1 -- 172.21.15.90:0/1961013791 learned_addr learned my addr 172.21.15.90:0/1961013791 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f323ce4b700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 0x559ab85c01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32377fe700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b9120 0x559ab85bfb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 -- 172.21.15.90:0/1961013791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b9120 msgr2=0x559ab85bfb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b9120 0x559ab85bfb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 -- 172.21.15.90:0/1961013791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 msgr2=0x559ab85c01d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 0x559ab85c01d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 -- 172.21.15.90:0/1961013791 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f322c01e040 con 0x559ab85b8800 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f323ce4b700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 0x559ab85c01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f3237fff700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f322809a2b0 tx=0x7f322809a320 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5426e25539108a22 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32177fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322809a780 con 0x559ab85b8800 2024-04-06T14:11:37.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559ab84ca400 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f3228099810 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f32280999d0 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f324aa62b80 10 client.26195 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559ab8337be0 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.722+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559ab84ca400 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3228025c90 con 0x559ab85b8800 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 10 client.26195 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 10 client.26195 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 20 client.26195 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab85d2a00 0x559ab85d4e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 4 client.26195 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559ab8502780 con 0x559ab85d2a00 2024-04-06T14:11:37.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.723+0000 7f32377fe700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab85d2a00 0x559ab85d4e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.724+0000 7f32177fe700 10 client.26195 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.724+0000 7f32377fe700 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab85d2a00 0x559ab85d4e20 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=38c25d24ddd99937 server_cookie=d7f2bed9a3d82704 in_seq=0 out_seq=0 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.724+0000 7f32177fe700 1 -- 172.21.15.90:0/1961013791 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x559ab8502780 con 0x559ab85d2a00 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.724+0000 7f32177fe700 10 client.26195 handle_command_reply: tid=0 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.724+0000 7f324aa62b80 1 client.26195 shutdown 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 msgr2=0x559ab85c2860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f322809a2b0 tx=0x7f322809a320 comp rx=0 tx=0).stop 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 shutdown_connections 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85c2e60 0x559ab85c01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b8800 0x559ab85c2860 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab85d2a00 0x559ab85d4e20 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1961013791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b9120 0x559ab85bfb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 >> 172.21.15.90:0/1961013791 conn(0x559ab8517720 msgr2=0x559ab85b2ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f32177fe700 0 client.26195 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:37.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 shutdown_connections 2024-04-06T14:11:37.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 1 -- 172.21.15.90:0/1961013791 wait complete. 2024-04-06T14:11:37.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.725+0000 7f324aa62b80 20 client.26195 trim_cache size 0 max 16384 2024-04-06T14:11:37.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 Processor -- start 2024-04-06T14:11:37.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 -- start start 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85beaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c9a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab85c0ae0 con 0x559ab85b8800 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab85c9fc0 con 0x559ab85b9120 2024-04-06T14:11:37.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.750+0000 7f324aa62b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab85b5270 con 0x559ab85c2e60 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f323ce4b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c9a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60328/0 (socket says 172.21.15.90:60328) 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 -- 172.21.15.90:0/2288788094 learned_addr learned my addr 172.21.15.90:0/2288788094 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 -- 172.21.15.90:0/2288788094 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 msgr2=0x559ab85c9a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c9a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 -- 172.21.15.90:0/2288788094 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 msgr2=0x559ab85beaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85beaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.751+0000 7f32377fe700 1 -- 172.21.15.90:0/2288788094 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559ab84cff20 con 0x559ab85b9120 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.752+0000 7f32377fe700 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f322c00dc70 tx=0x7f322c029000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2769bc16ffd0cf60 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.752+0000 7f321effd700 1 -- 172.21.15.90:0/2288788094 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322c032040 con 0x559ab85b9120 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.752+0000 7f321effd700 1 -- 172.21.15.90:0/2288788094 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f322c03c070 con 0x559ab85b9120 2024-04-06T14:11:37.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.752+0000 7f321effd700 1 -- 172.21.15.90:0/2288788094 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f322c0038e0 con 0x559ab85b9120 2024-04-06T14:11:37.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.753+0000 7f324aa62b80 1 -- 172.21.15.90:0/2288788094 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 msgr2=0x559ab85cea60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.753+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f322c00dc70 tx=0x7f322c029000 comp rx=0 tx=0).stop 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 -- 172.21.15.90:0/2288788094 shutdown_connections 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85c2e60 0x559ab85c9a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b8800 0x559ab85beaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 --2- 172.21.15.90:0/2288788094 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85b9120 0x559ab85cea60 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 -- 172.21.15.90:0/2288788094 >> 172.21.15.90:0/2288788094 conn(0x559ab8517720 msgr2=0x559ab85b77f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 -- 172.21.15.90:0/2288788094 shutdown_connections 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.755+0000 7f324aa62b80 1 -- 172.21.15.90:0/2288788094 wait complete. 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 -- start start 2024-04-06T14:11:37.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 Processor -- start 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b9120 0x559ab861ccd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85c2e60 0x559ab861d210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab8627810 con 0x559ab85b9120 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab8627950 con 0x559ab85c2e60 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab8627a90 con 0x559ab85b8800 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f324aa62b80 10 client.0 _ll_register_callbacks cb 0x559ab84af4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f3237fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f3237fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:53178/0 (socket says 172.21.15.90:53178) 2024-04-06T14:11:37.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.756+0000 7f3237fff700 1 -- 172.21.15.90:0/1389363455 learned_addr learned my addr 172.21.15.90:0/1389363455 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f323ce4b700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85c2e60 0x559ab861d210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32377fe700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b9120 0x559ab861ccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 -- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85c2e60 msgr2=0x559ab861d210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85c2e60 0x559ab861d210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 -- 172.21.15.90:0/1389363455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b9120 msgr2=0x559ab861ccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b9120 0x559ab861ccd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 -- 172.21.15.90:0/1389363455 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559ab838b430 con 0x559ab85b8800 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f3237fff700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f32280a2270 tx=0x7f32280caef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d995a86e7a78fa0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32177fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f32280961b0 con 0x559ab85b8800 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f322c0043c0 con 0x559ab85b8800 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f32280a2520 con 0x559ab85b8800 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3228096660 con 0x559ab85b8800 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f324aa62b80 10 client.26077 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:37.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559ab838fd50 con 0x559ab85b8800 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.757+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f322c0043c0 con 0x559ab85b8800 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3228099280 con 0x559ab85b8800 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 10 client.26077 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 10 client.26077 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 20 client.26077 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab8628ff0 0x559ab862b410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 4 client.26077 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x559ab8358d70 con 0x559ab8628ff0 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f32377fe700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab8628ff0 0x559ab862b410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f32177fe700 10 client.26077 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:37.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.758+0000 7f32377fe700 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab8628ff0 0x559ab862b410 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=65f8a034b3dadd84 server_cookie=99968ed3f09e2e5e in_seq=0 out_seq=0 2024-04-06T14:11:37.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.759+0000 7f32177fe700 1 -- 172.21.15.90:0/1389363455 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x559ab8358d70 con 0x559ab8628ff0 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.759+0000 7f32177fe700 10 client.26077 handle_command_reply: tid=0 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.759+0000 7f324aa62b80 1 client.26077 shutdown 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 msgr2=0x559ab861c790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f32280a2270 tx=0x7f32280caef0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 shutdown_connections 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab85c2e60 0x559ab861d210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab8628ff0 0x559ab862b410 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab85b8800 0x559ab861c790 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 --2- 172.21.15.90:0/1389363455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab85b9120 0x559ab861ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f32177fe700 0 client.26077 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 >> 172.21.15.90:0/1389363455 conn(0x559ab8517720 msgr2=0x559ab85c13f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 shutdown_connections 2024-04-06T14:11:37.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 1 -- 172.21.15.90:0/1389363455 wait complete. 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.762+0000 7f324aa62b80 20 client.26077 trim_cache size 0 max 16384 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "63949581-fd94-458b-9170-2ca653cd0417", 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:37.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 msgr2=0x7f32200b47e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 0x7f32200b47e0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f322c021000 tx=0x7f322c023000 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 msgr2=0x7f3238194f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f3228007bd0 tx=0x7f3228007f90 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 shutdown_connections 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f32200b2320 0x7f32200b47e0 secure :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f322c021000 tx=0x7f322c023000 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f32381997e0 0x7f3238199c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f32380ff5e0 0x7f3238194f00 secure :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f3228007bd0 tx=0x7f3228007f90 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 --2- 172.21.15.90:0/2373726931 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3238104700 0x7f3238195440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 >> 172.21.15.90:0/2373726931 conn(0x7f32380761b0 msgr2=0x7f3238102920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 shutdown_connections 2024-04-06T14:11:37.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:37.763+0000 7f323e8ae700 1 -- 172.21.15.90:0/2373726931 wait complete. 2024-04-06T14:11:37.784 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:11:37.785 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:11:38.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.017+0000 7f1898249700 1 -- 172.21.15.90:0/3824067541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 msgr2=0x7f1890107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.017+0000 7f1898249700 1 --2- 172.21.15.90:0/3824067541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f1890107db0 secure :-1 s=READY pgs=601 cs=0 l=1 rev1=1 crypto rx=0x7f188800b0a0 tx=0x7f188800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 350.0 s, 2184.2 tps, lat 45.363 ms stddev 49.965 2024-04-06T14:11:38.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.025+0000 7f1898249700 1 -- 172.21.15.90:0/3824067541 shutdown_connections 2024-04-06T14:11:38.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.025+0000 7f1898249700 1 --2- 172.21.15.90:0/3824067541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f189011a590 0x7f189011c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.025+0000 7f1898249700 1 --2- 172.21.15.90:0/3824067541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f1890107db0 unknown :-1 s=CLOSED pgs=601 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.025+0000 7f1898249700 1 --2- 172.21.15.90:0/3824067541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f189010f920 0x7f18901082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.025+0000 7f1898249700 1 -- 172.21.15.90:0/3824067541 >> 172.21.15.90:0/3824067541 conn(0x7f189006be00 msgr2=0x7f189006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.030+0000 7f1898249700 1 -- 172.21.15.90:0/3824067541 shutdown_connections 2024-04-06T14:11:38.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.033+0000 7f1898249700 1 -- 172.21.15.90:0/3824067541 wait complete. 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 Processor -- start 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 -- start start 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f189010f920 0x7f18901a6610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f189011a590 0x7f18901aa9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18901aaff0 con 0x7f189010f4b0 2024-04-06T14:11:38.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18901ab160 con 0x7f189011a590 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1898249700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18901ab320 con 0x7f189010f920 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1895fe5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1895fe5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59744/0 (socket says 172.21.15.90:59744) 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.034+0000 7f1895fe5700 1 -- 172.21.15.90:0/2889674551 learned_addr learned my addr 172.21.15.90:0/2889674551 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 -- 172.21.15.90:0/2889674551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f189010f920 msgr2=0x7f18901a6610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f189010f920 0x7f18901a6610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 -- 172.21.15.90:0/2889674551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f189011a590 msgr2=0x7f18901aa9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f189011a590 0x7f18901aa9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 -- 172.21.15.90:0/2889674551 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1888009d00 con 0x7f189010f4b0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1895fe5700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f188800b070 tx=0x7f1888004d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1886ffd700 1 -- 172.21.15.90:0/2889674551 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1888021070 con 0x7f189010f4b0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.035+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18901ab5a0 con 0x7f189010f4b0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.036+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18901b2e40 con 0x7f189010f4b0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.036+0000 7f1886ffd700 1 -- 172.21.15.90:0/2889674551 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1888009450 con 0x7f189010f4b0 2024-04-06T14:11:38.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.036+0000 7f1886ffd700 1 -- 172.21.15.90:0/2889674551 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188800e040 con 0x7f189010f4b0 2024-04-06T14:11:38.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.038+0000 7f1886ffd700 1 -- 172.21.15.90:0/2889674551 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1888004620 con 0x7f189010f4b0 2024-04-06T14:11:38.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.039+0000 7f1886ffd700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 0x7f187c0a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.039+0000 7f1886ffd700 1 -- 172.21.15.90:0/2889674551 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f18880ce860 con 0x7f189010f4b0 2024-04-06T14:11:38.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.041+0000 7f18957e4700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 0x7f187c0a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 Processor -- start 2024-04-06T14:11:38.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18957e4700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 0x7f187c0a3810 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f1880005950 tx=0x7f18800058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 -- start start 2024-04-06T14:11:38.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7414b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a74211d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a736c990 con 0x5580a74150a0 2024-04-06T14:11:38.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a736a350 con 0x5580a7414780 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7365c40 con 0x5580a741ede0 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18957e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18957e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741e8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59752/0 (socket says 172.21.15.90:59752) 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f18957e4700 1 -- 172.21.15.90:0/777685757 learned_addr learned my addr 172.21.15.90:0/777685757 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.043+0000 7f1895fe5700 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7414b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 -- 172.21.15.90:0/777685757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 msgr2=0x5580a74211d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a74211d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 -- 172.21.15.90:0/777685757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 msgr2=0x5580a741e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741e8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 -- 172.21.15.90:0/777685757 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5580a71e73b0 con 0x5580a7414780 2024-04-06T14:11:38.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f1895fe5700 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7414b60 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f18880cc050 tx=0x7f1888097ef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=620a43d317ae40cd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f187bfff700 1 -- 172.21.15.90:0/777685757 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188809baa0 con 0x5580a7414780 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f187bfff700 1 -- 172.21.15.90:0/777685757 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f188809f8d0 con 0x5580a7414780 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.044+0000 7f187bfff700 1 -- 172.21.15.90:0/777685757 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188809bd00 con 0x5580a7414780 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.046+0000 7f18a43fdb80 1 -- 172.21.15.90:0/777685757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 msgr2=0x5580a7414b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.046+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7414b60 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f18880cc050 tx=0x7f1888097ef0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 -- 172.21.15.90:0/777685757 shutdown_connections 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a74211d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7414b60 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/777685757 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 -- 172.21.15.90:0/777685757 >> 172.21.15.90:0/777685757 conn(0x5580a73736a0 msgr2=0x5580a740fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 -- 172.21.15.90:0/777685757 shutdown_connections 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.047+0000 7f18a43fdb80 1 -- 172.21.15.90:0/777685757 wait complete. 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 -- start start 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 Processor -- start 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741e810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741bb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7301d80 con 0x5580a74150a0 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7301a20 con 0x5580a741ede0 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a71e96a0 con 0x5580a7414780 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.048+0000 7f18a43fdb80 10 client.0 _ll_register_callbacks cb 0x5580a730b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60362/0 (socket says 172.21.15.90:60362) 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 -- 172.21.15.90:0/2590824578 learned_addr learned my addr 172.21.15.90:0/2590824578 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18957e4700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 -- 172.21.15.90:0/2590824578 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 msgr2=0x5580a741e810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741e810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 -- 172.21.15.90:0/2590824578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 msgr2=0x5580a741bb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741bb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 -- 172.21.15.90:0/2590824578 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18901b2e40 con 0x5580a741ede0 2024-04-06T14:11:38.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18957e4700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18967e6700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f188c00e600 tx=0x7f188c00e9c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=de71d5f90d69fb49 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18797fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188c0096e0 con 0x5580a741ede0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f188c014070 con 0x5580a741ede0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.049+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5580a7326380 con 0x5580a741ede0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f188c004a30 con 0x5580a741ede0 2024-04-06T14:11:38.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 10 client.26210 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5580a7193b60 con 0x5580a741ede0 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5580a7326380 con 0x5580a741ede0 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f188c010850 con 0x5580a741ede0 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 10 client.26210 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 10 client.26210 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 20 client.26210 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a743f6e0 0x5580a7441b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 4 client.26210 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5580a735e700 con 0x5580a743f6e0 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.050+0000 7f1895fe5700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a743f6e0 0x5580a7441b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.051+0000 7f18797fa700 10 client.26210 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.051+0000 7f1895fe5700 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a743f6e0 0x5580a7441b00 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=45cd196ea14e33a3 server_cookie=f8b1ae6d4fa24563 in_seq=0 out_seq=0 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.052+0000 7f18797fa700 1 -- 172.21.15.90:0/2590824578 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5580a735e700 con 0x5580a743f6e0 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.052+0000 7f18797fa700 10 client.26210 handle_command_reply: tid=0 2024-04-06T14:11:38.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.052+0000 7f18a43fdb80 1 client.26210 shutdown 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 msgr2=0x5580a741c290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f188c00e600 tx=0x7f188c00e9c0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 shutdown_connections 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a741c290 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a743f6e0 0x5580a7441b00 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741e810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/2590824578 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a741bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 >> 172.21.15.90:0/2590824578 conn(0x5580a73736a0 msgr2=0x5580a740ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18797fa700 0 client.26210 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:38.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 shutdown_connections 2024-04-06T14:11:38.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 1 -- 172.21.15.90:0/2590824578 wait complete. 2024-04-06T14:11:38.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.053+0000 7f18a43fdb80 20 client.26210 trim_cache size 0 max 16384 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.076+0000 7f18a43fdb80 1 Processor -- start 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.076+0000 7f18a43fdb80 1 -- start start 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a740b890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a7436750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a740bdd0 con 0x5580a74150a0 2024-04-06T14:11:38.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7436c90 con 0x5580a741ede0 2024-04-06T14:11:38.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a743ba70 con 0x5580a7414780 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f1895fe5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f1895fe5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:53230/0 (socket says 172.21.15.90:53230) 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f1895fe5700 1 -- 172.21.15.90:0/3171552174 learned_addr learned my addr 172.21.15.90:0/3171552174 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18967e6700 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a7436750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.077+0000 7f18957e4700 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a740b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 -- 172.21.15.90:0/3171552174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 msgr2=0x5580a7436750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a7436750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 -- 172.21.15.90:0/3171552174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 msgr2=0x5580a740b890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a740b890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 -- 172.21.15.90:0/3171552174 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5580a732bea0 con 0x5580a7414780 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1895fe5700 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f18880cc0b0 tx=0x7f1888009a50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=73696595cb74ddfd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1884ff9700 1 -- 172.21.15.90:0/3171552174 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188809baa0 con 0x5580a7414780 2024-04-06T14:11:38.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1884ff9700 1 -- 172.21.15.90:0/3171552174 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f18880a3e90 con 0x5580a7414780 2024-04-06T14:11:38.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.078+0000 7f1884ff9700 1 -- 172.21.15.90:0/3171552174 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f188809af90 con 0x5580a7414780 2024-04-06T14:11:38.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- 172.21.15.90:0/3171552174 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 msgr2=0x5580a741b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f18880cc0b0 tx=0x7f1888009a50 comp rx=0 tx=0).stop 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- 172.21.15.90:0/3171552174 shutdown_connections 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a741ede0 0x5580a7436750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a7414780 0x5580a741b430 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/3171552174 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a740b890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- 172.21.15.90:0/3171552174 >> 172.21.15.90:0/3171552174 conn(0x5580a73736a0 msgr2=0x5580a7446460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- 172.21.15.90:0/3171552174 shutdown_connections 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- 172.21.15.90:0/3171552174 wait complete. 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.079+0000 7f18a43fdb80 1 -- start start 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 Processor -- start 2024-04-06T14:11:38.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7436400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a7434de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7435320 con 0x5580a74150a0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7435490 con 0x5580a7414780 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5580a7435650 con 0x5580a741ede0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18a43fdb80 10 client.0 _ll_register_callbacks cb 0x5580a730b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18957e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18957e4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59764/0 (socket says 172.21.15.90:59764) 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18967e6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a7434de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f18957e4700 1 -- 172.21.15.90:0/437964117 learned_addr learned my addr 172.21.15.90:0/437964117 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.080+0000 7f1895fe5700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7436400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 -- 172.21.15.90:0/437964117 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 msgr2=0x5580a7434de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a7434de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 -- 172.21.15.90:0/437964117 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 msgr2=0x5580a74348a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 -- 172.21.15.90:0/437964117 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5580a71e73b0 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18967e6700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a7434de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18957e4700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f1895fe5700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7436400 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1888004e50 tx=0x7f188800b7b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=93b3788d8893a473 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18797fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18880079d0 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(84..84 src has 1..84) v4 ==== 8512+0+0 (secure 0 0 0) 0x7f1888099f00 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f188809f8d0 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f188809bd00 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18a43fdb80 10 client.26216 fetch_fsmap learned FSMap version 24 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5580a71ebcd0 con 0x5580a7414780 2024-04-06T14:11:38.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.081+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f188809f8d0 con 0x5580a7414780 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f188809af90 con 0x5580a7414780 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 10 client.26216 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 10 client.26216 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 20 client.26216 populate_metadata read hostname 'smithi090' 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a74945a0 0x5580a74969c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 4 client.26216 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5580a71b4cf0 con 0x5580a74945a0 2024-04-06T14:11:38.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.082+0000 7f18957e4700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a74945a0 0x5580a74969c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:11:38.086 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18957e4700 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a74945a0 0x5580a74969c0 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=820cb12316526e0f server_cookie=3ff1442cf1a180e5 in_seq=0 out_seq=0 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18797fa700 10 client.26216 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18797fa700 1 -- 172.21.15.90:0/437964117 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x5580a71b4cf0 con 0x5580a74945a0 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18797fa700 10 client.26216 handle_command_reply: tid=0 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18a43fdb80 1 client.26216 shutdown 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 msgr2=0x5580a7436400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.083+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7436400 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1888004e50 tx=0x7f188800b7b0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 shutdown_connections 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5580a74945a0 0x5580a74969c0 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5580a741ede0 0x5580a7434de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5580a7414780 0x5580a7436400 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 --2- 172.21.15.90:0/437964117 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5580a74150a0 0x5580a74348a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.087 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 >> 172.21.15.90:0/437964117 conn(0x5580a73736a0 msgr2=0x5580a7413770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18797fa700 0 client.26216 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 shutdown_connections 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 1 -- 172.21.15.90:0/437964117 wait complete. 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.084+0000 7f18a43fdb80 20 client.26216 trim_cache size 0 max 16384 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (221 inodes in the stack)", 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout: "63949581-fd94-458b-9170-2ca653cd0417": { 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:11:38.088 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "63949581-fd94-458b-9170-2ca653cd0417", 2024-04-06T14:11:38.089 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:11:38.089 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:11:38.089 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:11:38.089 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:11:38.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.085+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 msgr2=0x7f187c0a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.085+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 0x7f187c0a3810 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f1880005950 tx=0x7f18800058e0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.085+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 msgr2=0x7f18901a60d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.085+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 secure :-1 s=READY pgs=602 cs=0 l=1 rev1=1 crypto rx=0x7f188800b070 tx=0x7f1888004d30 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 shutdown_connections 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f187c0a1350 0x7f187c0a3810 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f189011a590 0x7f18901aa9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f189010f4b0 0x7f18901a60d0 unknown :-1 s=CLOSED pgs=602 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 --2- 172.21.15.90:0/2889674551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f189010f920 0x7f18901a6610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.087+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 >> 172.21.15.90:0/2889674551 conn(0x7f189006be00 msgr2=0x7f1890118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.088+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 shutdown_connections 2024-04-06T14:11:38.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:11:38.088+0000 7f1898249700 1 -- 172.21.15.90:0/2889674551 wait complete. 2024-04-06T14:11:38.103 INFO:tasks.cephfs.filesystem:scrub status for tag:63949581-fd94-458b-9170-2ca653cd0417 - {'path': '/', 'tag': '63949581-fd94-458b-9170-2ca653cd0417', 'options': 'recursive,force'} 2024-04-06T14:11:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:38 smithi105 ceph-mon[46969]: pgmap v190: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:38 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:11:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:38 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:38 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:11:38.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:38 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:39.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:38 smithi090 ceph-mon[43420]: pgmap v190: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:39.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:38 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:11:39.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:38 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:39.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:38 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:11:39.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:38 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:39.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:38 smithi134 ceph-mon[47549]: pgmap v190: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:39.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:38 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:11:39.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:38 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:39.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:38 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:11:39.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:38 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:11:41.046 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:40 smithi090 ceph-mon[43420]: pgmap v191: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:41.046 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:40 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:11:41.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:40 smithi134 ceph-mon[47549]: pgmap v191: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:40 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:11:41.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:40 smithi105 ceph-mon[46969]: pgmap v191: 97 pgs: 97 active+clean; 8.0 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 9.7 MiB/s wr, 651 op/s 2024-04-06T14:11:41.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:40 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:11:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 355.0 s, 1522.9 tps, lat 66.045 ms stddev 74.774 2024-04-06T14:11:43.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:42 smithi090 ceph-mon[43420]: pgmap v192: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 15 MiB/s wr, 987 op/s 2024-04-06T14:11:43.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:42 smithi134 ceph-mon[47549]: pgmap v192: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 15 MiB/s wr, 987 op/s 2024-04-06T14:11:43.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:42 smithi105 ceph-mon[46969]: pgmap v192: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 15 MiB/s wr, 987 op/s 2024-04-06T14:11:45.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:44 smithi090 ceph-mon[43420]: pgmap v193: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 9.8 MiB/s wr, 624 op/s 2024-04-06T14:11:45.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:44 smithi134 ceph-mon[47549]: pgmap v193: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 9.8 MiB/s wr, 624 op/s 2024-04-06T14:11:45.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:44 smithi105 ceph-mon[46969]: pgmap v193: 97 pgs: 97 active+clean; 8.1 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 9.8 MiB/s wr, 624 op/s 2024-04-06T14:11:47.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:46 smithi090 ceph-mon[43420]: pgmap v194: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 18 MiB/s wr, 990 op/s 2024-04-06T14:11:47.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:46 smithi134 ceph-mon[47549]: pgmap v194: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 18 MiB/s wr, 990 op/s 2024-04-06T14:11:47.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:46 smithi105 ceph-mon[46969]: pgmap v194: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 18 MiB/s wr, 990 op/s 2024-04-06T14:11:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 360.0 s, 2212.6 tps, lat 45.492 ms stddev 56.915 2024-04-06T14:11:48.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:48 smithi105 ceph-mon[46969]: pgmap v195: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:48 smithi090 ceph-mon[43420]: pgmap v195: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:49.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:48 smithi134 ceph-mon[47549]: pgmap v195: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:51.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:50 smithi090 ceph-mon[43420]: pgmap v196: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:51.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:50 smithi134 ceph-mon[47549]: pgmap v196: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:51.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:50 smithi105 ceph-mon[46969]: pgmap v196: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 13 MiB/s wr, 701 op/s 2024-04-06T14:11:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 365.0 s, 2177.0 tps, lat 45.544 ms stddev 52.983 2024-04-06T14:11:53.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:52 smithi090 ceph-mon[43420]: pgmap v197: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 21 MiB/s wr, 993 op/s 2024-04-06T14:11:53.147 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:52 smithi134 ceph-mon[47549]: pgmap v197: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 21 MiB/s wr, 993 op/s 2024-04-06T14:11:53.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:52 smithi105 ceph-mon[46969]: pgmap v197: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 21 MiB/s wr, 993 op/s 2024-04-06T14:11:55.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:54 smithi090 ceph-mon[43420]: pgmap v198: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 658 op/s 2024-04-06T14:11:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:54 smithi134 ceph-mon[47549]: pgmap v198: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 658 op/s 2024-04-06T14:11:55.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:54 smithi105 ceph-mon[46969]: pgmap v198: 97 pgs: 97 active+clean; 8.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 658 op/s 2024-04-06T14:11:56.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:55 smithi134 ceph-mon[47549]: pgmap v199: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 24 MiB/s wr, 1.14k op/s 2024-04-06T14:11:56.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:55 smithi105 ceph-mon[46969]: pgmap v199: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 24 MiB/s wr, 1.14k op/s 2024-04-06T14:11:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:55 smithi090 ceph-mon[43420]: pgmap v199: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 24 MiB/s wr, 1.14k op/s 2024-04-06T14:11:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 370.0 s, 980.0 tps, lat 103.194 ms stddev 213.415 2024-04-06T14:11:58.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:11:58 smithi105 ceph-mon[46969]: pgmap v200: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 16 MiB/s wr, 772 op/s 2024-04-06T14:11:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:11:58 smithi090 ceph-mon[43420]: pgmap v200: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 16 MiB/s wr, 772 op/s 2024-04-06T14:11:59.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:11:58 smithi134 ceph-mon[47549]: pgmap v200: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 16 MiB/s wr, 772 op/s 2024-04-06T14:12:01.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:00 smithi134 ceph-mon[47549]: pgmap v201: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 19 MiB/s wr, 901 op/s 2024-04-06T14:12:01.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:00 smithi105 ceph-mon[46969]: pgmap v201: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 19 MiB/s wr, 901 op/s 2024-04-06T14:12:01.294 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:00 smithi090 ceph-mon[43420]: pgmap v201: 97 pgs: 97 active+clean; 8.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 19 MiB/s wr, 901 op/s 2024-04-06T14:12:02.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:01 smithi134 ceph-mon[47549]: pgmap v202: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 154 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:12:02.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:01 smithi105 ceph-mon[46969]: pgmap v202: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 154 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:12:02.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:01 smithi090 ceph-mon[43420]: pgmap v202: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 154 KiB/s rd, 45 MiB/s wr, 2.21k op/s 2024-04-06T14:12:03.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 375.0 s, 2082.2 tps, lat 47.860 ms stddev 104.989 2024-04-06T14:12:05.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:04 smithi134 ceph-mon[47549]: pgmap v203: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 137 KiB/s rd, 36 MiB/s wr, 1.92k op/s 2024-04-06T14:12:05.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:04 smithi105 ceph-mon[46969]: pgmap v203: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 137 KiB/s rd, 36 MiB/s wr, 1.92k op/s 2024-04-06T14:12:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:04 smithi090 ceph-mon[43420]: pgmap v203: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 137 KiB/s rd, 36 MiB/s wr, 1.92k op/s 2024-04-06T14:12:07.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:06 smithi134 ceph-mon[47549]: pgmap v204: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 182 KiB/s rd, 47 MiB/s wr, 2.73k op/s 2024-04-06T14:12:07.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:06 smithi134 ceph-mon[47549]: osdmap e85: 12 total, 12 up, 12 in 2024-04-06T14:12:07.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:06 smithi105 ceph-mon[46969]: pgmap v204: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 182 KiB/s rd, 47 MiB/s wr, 2.73k op/s 2024-04-06T14:12:07.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:06 smithi105 ceph-mon[46969]: osdmap e85: 12 total, 12 up, 12 in 2024-04-06T14:12:07.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:06 smithi090 ceph-mon[43420]: pgmap v204: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 182 KiB/s rd, 47 MiB/s wr, 2.73k op/s 2024-04-06T14:12:07.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:06 smithi090 ceph-mon[43420]: osdmap e85: 12 total, 12 up, 12 in 2024-04-06T14:12:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 380.0 s, 1908.6 tps, lat 52.349 ms stddev 64.083 2024-04-06T14:12:08.103 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:12:08.104 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:12:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.323+0000 7f839e097700 1 -- 172.21.15.90:0/1656657871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 msgr2=0x7f839811c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.323+0000 7f839e097700 1 --2- 172.21.15.90:0/1656657871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f839811c980 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f8388009990 tx=0x7f8388009ca0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.324+0000 7f839e097700 1 -- 172.21.15.90:0/1656657871 shutdown_connections 2024-04-06T14:12:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.324+0000 7f839e097700 1 --2- 172.21.15.90:0/1656657871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f839811c980 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.324+0000 7f839e097700 1 --2- 172.21.15.90:0/1656657871 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f4b0 0x7f8398107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.324+0000 7f839e097700 1 --2- 172.21.15.90:0/1656657871 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f839810f920 0x7f83981082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.324+0000 7f839e097700 1 -- 172.21.15.90:0/1656657871 >> 172.21.15.90:0/1656657871 conn(0x7f839806be00 msgr2=0x7f839806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.327+0000 7f839e097700 1 -- 172.21.15.90:0/1656657871 shutdown_connections 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.328+0000 7f839e097700 1 -- 172.21.15.90:0/1656657871 wait complete. 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.329+0000 7f839e097700 1 Processor -- start 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 -- start start 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f839810f4b0 0x7f8398118a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f920 0x7f8398118fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83981123d0 con 0x7f839810f4b0 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8398112540 con 0x7f839811a590 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f839e097700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8398112700 con 0x7f839810f920 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8396ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f920 0x7f8398118fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53894/0 (socket says 172.21.15.90:53894) 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 -- 172.21.15.90:0/1729622604 learned_addr learned my addr 172.21.15.90:0/1729622604 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 -- 172.21.15.90:0/1729622604 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f920 msgr2=0x7f8398118fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f920 0x7f8398118fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 -- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f839810f4b0 msgr2=0x7f8398118a90 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f839810f4b0 0x7f8398118a90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.330+0000 7f8397fff700 1 -- 172.21.15.90:0/1729622604 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83880095f0 con 0x7f839811a590 2024-04-06T14:12:08.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.333+0000 7f8397fff700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f8388005b40 tx=0x7f8388003b60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.335+0000 7f8394ff9700 1 -- 172.21.15.90:0/1729622604 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838801d070 con 0x7f839811a590 2024-04-06T14:12:08.336 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.335+0000 7f8394ff9700 1 -- 172.21.15.90:0/1729622604 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8388005580 con 0x7f839811a590 2024-04-06T14:12:08.336 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.335+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83981165b0 con 0x7f839811a590 2024-04-06T14:12:08.336 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.335+0000 7f8394ff9700 1 -- 172.21.15.90:0/1729622604 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838800fdc0 con 0x7f839811a590 2024-04-06T14:12:08.336 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.335+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8398116a60 con 0x7f839811a590 2024-04-06T14:12:08.341 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 Processor -- start 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 -- start start 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca54b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 0x563acca5e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca5ede0 0x563acca611d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc9ac990 con 0x563acca54780 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc9aa350 con 0x563acca550a0 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.341+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc9a5c40 con 0x563acca5ede0 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f83977fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 0x563acca5e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.343 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f83977fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 0x563acca5e8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53904/0 (socket says 172.21.15.90:53904) 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f83977fe700 1 -- 172.21.15.90:0/3242446342 learned_addr learned my addr 172.21.15.90:0/3242446342 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca54b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca5ede0 0x563acca611d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8394ff9700 1 -- 172.21.15.90:0/1729622604 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f83880050c0 con 0x7f839811a590 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 -- 172.21.15.90:0/3242446342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca5ede0 msgr2=0x563acca611d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca5ede0 0x563acca611d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 -- 172.21.15.90:0/3242446342 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 msgr2=0x563acca5e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 0x563acca5e8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.342+0000 7f8396ffd700 1 -- 172.21.15.90:0/3242446342 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563acc8273b0 con 0x563acca54780 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.343+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca54b60 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f83981132d0 tx=0x7f8390009b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c90eef3dc2c0ff00 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.343+0000 7f837dffb700 1 -- 172.21.15.90:0/3242446342 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8390008a80 con 0x563acca54780 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.343+0000 7f837dffb700 1 -- 172.21.15.90:0/3242446342 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8390008be0 con 0x563acca54780 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.343+0000 7f837dffb700 1 -- 172.21.15.90:0/3242446342 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8390019380 con 0x563acca54780 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.344+0000 7f8394ff9700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 0x7f83800a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.344+0000 7f8394ff9700 1 -- 172.21.15.90:0/1729622604 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f83880cdee0 con 0x7f839811a590 2024-04-06T14:12:08.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.344+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3242446342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 msgr2=0x563acca54b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.344+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca54b60 secure :-1 s=READY pgs=603 cs=0 l=1 rev1=1 crypto rx=0x7f83981132d0 tx=0x7f8390009b50 comp rx=0 tx=0).stop 2024-04-06T14:12:08.346 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.344+0000 7f83977fe700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 0x7f83800a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.345+0000 7f83977fe700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 0x7f83800a3920 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f838c009a50 tx=0x7f838c009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3242446342 shutdown_connections 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca5ede0 0x563acca611d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca54b60 unknown :-1 s=CLOSED pgs=603 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3242446342 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca550a0 0x563acca5e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3242446342 >> 172.21.15.90:0/3242446342 conn(0x563acc9b36a0 msgr2=0x563acca4fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3242446342 shutdown_connections 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3242446342 wait complete. 2024-04-06T14:12:08.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.346+0000 7f83aa24bb80 1 -- start start 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 Processor -- start 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca5e7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca5bb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc941d80 con 0x563acca54780 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc941a20 con 0x563acca5ede0 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acc8296a0 con 0x563acca550a0 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f83aa24bb80 10 client.0 _ll_register_callbacks cb 0x563acc94b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:08.349 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f8396ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca5bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53920/0 (socket says 172.21.15.90:53920) 2024-04-06T14:12:08.350 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.347+0000 7f8397fff700 1 -- 172.21.15.90:0/1316272338 learned_addr learned my addr 172.21.15.90:0/1316272338 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 -- 172.21.15.90:0/1316272338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 msgr2=0x563acca5bb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca5bb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 -- 172.21.15.90:0/1316272338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 msgr2=0x563acca5e7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca5e7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 -- 172.21.15.90:0/1316272338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f839000ba20 con 0x563acca5ede0 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8396ffd700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca5bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8397fff700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f83880a2e90 tx=0x7f838809a9b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e5250137ac557db9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8376ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:08.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838809b6d0 con 0x563acca5ede0 2024-04-06T14:12:08.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563acc966380 con 0x563acca5ede0 2024-04-06T14:12:08.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f838809afd0 con 0x563acca5ede0 2024-04-06T14:12:08.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f838809ee00 con 0x563acca5ede0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f83aa24bb80 10 client.26228 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563acc7d3b60 con 0x563acca5ede0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.348+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f83880a7080 con 0x563acca5ede0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f838809fd20 con 0x563acca5ede0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 10 client.26228 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 10 client.26228 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 20 client.26228 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563acca6e4c0 0x563acca708e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 4 client.26228 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563acc99e700 con 0x563acca6e4c0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83977fe700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563acca6e4c0 0x563acca708e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f83977fe700 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563acca6e4c0 0x563acca708e0 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=457e816352c85d7f server_cookie=3a8467672b3099be in_seq=0 out_seq=0 2024-04-06T14:12:08.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.349+0000 7f8376ffd700 10 client.26228 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f8376ffd700 1 -- 172.21.15.90:0/1316272338 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563acc99e700 con 0x563acca6e4c0 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f8376ffd700 10 client.26228 handle_command_reply: tid=0 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 client.26228 shutdown 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 msgr2=0x563acca5c140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f83880a2e90 tx=0x7f838809a9b0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 shutdown_connections 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5c140 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca5e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563acca6e4c0 0x563acca708e0 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/1316272338 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca5bb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f8376ffd700 0 client.26228 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 >> 172.21.15.90:0/1316272338 conn(0x563acc9b36a0 msgr2=0x563acca4ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 shutdown_connections 2024-04-06T14:12:08.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 1 -- 172.21.15.90:0/1316272338 wait complete. 2024-04-06T14:12:08.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.350+0000 7f83aa24bb80 20 client.26228 trim_cache size 0 max 16384 2024-04-06T14:12:08.374 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.372+0000 7f83aa24bb80 1 Processor -- start 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 -- start start 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca65a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5e2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acca65f40 con 0x563acca54780 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acca5a0a0 con 0x563acca5ede0 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563acca57730 con 0x563acca550a0 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59102/0 (socket says 172.21.15.90:59102) 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 -- 172.21.15.90:0/3162008980 learned_addr learned my addr 172.21.15.90:0/3162008980 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f8397fff700 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5e2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f8396ffd700 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca65a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 -- 172.21.15.90:0/3162008980 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 msgr2=0x563acca65a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca65a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 -- 172.21.15.90:0/3162008980 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 msgr2=0x563acca5e2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5e2f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.373+0000 7f83977fe700 1 -- 172.21.15.90:0/3162008980 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563acc96bea0 con 0x563acca54780 2024-04-06T14:12:08.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.374+0000 7f83977fe700 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x563acca711c0 tx=0x7f838c004790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b9bb2813759e5bd7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.374+0000 7f837e7fc700 1 -- 172.21.15.90:0/3162008980 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838c023e70 con 0x563acca54780 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.374+0000 7f837e7fc700 1 -- 172.21.15.90:0/3162008980 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f838c027070 con 0x563acca54780 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.374+0000 7f837e7fc700 1 -- 172.21.15.90:0/3162008980 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838c0156e0 con 0x563acca54780 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3162008980 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 msgr2=0x563acca4fc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 secure :-1 s=READY pgs=604 cs=0 l=1 rev1=1 crypto rx=0x563acca711c0 tx=0x7f838c004790 comp rx=0 tx=0).stop 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3162008980 shutdown_connections 2024-04-06T14:12:08.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca5e2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca4fc30 unknown :-1 s=CLOSED pgs=604 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/3162008980 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca65a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3162008980 >> 172.21.15.90:0/3162008980 conn(0x563acc9b36a0 msgr2=0x563acca4f520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3162008980 shutdown_connections 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.375+0000 7f83aa24bb80 1 -- 172.21.15.90:0/3162008980 wait complete. 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.376+0000 7f83aa24bb80 1 -- start start 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.376+0000 7f83aa24bb80 1 Processor -- start 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.376+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca64010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.376+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca64550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.376+0000 7f83aa24bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563accac2300 con 0x563acca54780 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563accac2440 con 0x563acca5ede0 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f83aa24bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563accac2580 con 0x563acca550a0 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f83aa24bb80 10 client.0 _ll_register_callbacks cb 0x563acc94b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53946/0 (socket says 172.21.15.90:53946) 2024-04-06T14:12:08.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 -- 172.21.15.90:0/786673400 learned_addr learned my addr 172.21.15.90:0/786673400 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f83977fe700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca64010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8396ffd700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca64550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 -- 172.21.15.90:0/786673400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 msgr2=0x563acca64550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca64550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 -- 172.21.15.90:0/786673400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 msgr2=0x563acca64010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca64010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 -- 172.21.15.90:0/786673400 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f838c01a040 con 0x563acca5ede0 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8397fff700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f8388099320 tx=0x7f838800f670 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1de53a79f12fb595 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8376ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:08.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.377+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838809b430 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f838809b6d0 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8390008be0 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8388099da0 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f83aa24bb80 10 client.26234 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563acc82bcd0 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.378+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8390008be0 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8388029060 con 0x563acca5ede0 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 10 client.26234 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 10 client.26234 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 20 client.26234 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563accac3860 0x563accac5c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 4 client.26234 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x563acc7f4cf0 con 0x563accac3860 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83977fe700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563accac3860 0x563accac5c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f8376ffd700 10 client.26234 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.379+0000 7f83977fe700 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563accac3860 0x563accac5c50 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e593385532658531 server_cookie=57a0a3718813b3ca in_seq=0 out_seq=0 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f8376ffd700 1 -- 172.21.15.90:0/786673400 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x563acc7f4cf0 con 0x563accac3860 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f8376ffd700 10 client.26234 handle_command_reply: tid=0 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 client.26234 shutdown 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 msgr2=0x563acca64a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f8388099320 tx=0x7f838800f670 comp rx=0 tx=0).stop 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 shutdown_connections 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563acca5ede0 0x563acca64a90 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563acca54780 0x563acca64010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563accac3860 0x563accac5c50 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 --2- 172.21.15.90:0/786673400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563acca550a0 0x563acca64550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f8376ffd700 0 client.26234 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:08.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.380+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 >> 172.21.15.90:0/786673400 conn(0x563acc9b36a0 msgr2=0x563acca54000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.381+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 shutdown_connections 2024-04-06T14:12:08.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.381+0000 7f83aa24bb80 1 -- 172.21.15.90:0/786673400 wait complete. 2024-04-06T14:12:08.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f83aa24bb80 20 client.26234 trim_cache size 0 max 16384 2024-04-06T14:12:08.395 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:12:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:12:08.396 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:08.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 msgr2=0x7f83800a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 0x7f83800a3920 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f838c009a50 tx=0x7f838c009190 comp rx=0 tx=0).stop 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 msgr2=0x7f8398111c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f8388005b40 tx=0x7f8388003b60 comp rx=0 tx=0).stop 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 shutdown_connections 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83800a1460 0x7f83800a3920 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f839811a590 0x7f8398111c30 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f839810f4b0 0x7f8398118a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 --2- 172.21.15.90:0/1729622604 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f839810f920 0x7f8398118fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.382+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 >> 172.21.15.90:0/1729622604 conn(0x7f839806be00 msgr2=0x7f839806f7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.383+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 shutdown_connections 2024-04-06T14:12:08.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.383+0000 7f839e097700 1 -- 172.21.15.90:0/1729622604 wait complete. 2024-04-06T14:12:08.400 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:12:08.401 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:12:08.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 -- 172.21.15.90:0/3111136474 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08105bf0 msgr2=0x7efc08105fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/3111136474 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08105bf0 0x7efc08105fd0 secure :-1 s=READY pgs=605 cs=0 l=1 rev1=1 crypto rx=0x7efbf8009990 tx=0x7efbf8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 -- 172.21.15.90:0/3111136474 shutdown_connections 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/3111136474 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc0810c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/3111136474 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc08105bf0 0x7efc08105fd0 unknown :-1 s=CLOSED pgs=605 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/3111136474 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc080ffb30 0x7efc080fffb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.579+0000 7efc0d0d1700 1 -- 172.21.15.90:0/3111136474 >> 172.21.15.90:0/3111136474 conn(0x7efc080fb5e0 msgr2=0x7efc080fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.580+0000 7efc0d0d1700 1 -- 172.21.15.90:0/3111136474 shutdown_connections 2024-04-06T14:12:08.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.580+0000 7efc0d0d1700 1 -- 172.21.15.90:0/3111136474 wait complete. 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.580+0000 7efc0d0d1700 1 Processor -- start 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 -- start start 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc08195380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 0x7efc08199720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc08199d60 con 0x7efc080ffb30 2024-04-06T14:12:08.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc08199ed0 con 0x7efc08105bf0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0d0d1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc0819a090 con 0x7efc081004f0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc06d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc06d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59126/0 (socket says 172.21.15.90:59126) 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc06d9d700 1 -- 172.21.15.90:0/683498893 learned_addr learned my addr 172.21.15.90:0/683498893 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0759e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 0x7efc08199720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.581+0000 7efc0659c700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc08195380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 -- 172.21.15.90:0/683498893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 msgr2=0x7efc08195380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc08195380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 -- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 msgr2=0x7efc08194e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 -- 172.21.15.90:0/683498893 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efbfc009520 con 0x7efc08105bf0 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc06d9d700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:08.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0659c700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc08195380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0759e700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 0x7efc08199720 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7efc0000efc0 tx=0x7efc0000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc00018070 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc0000f040 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efbf80095f0 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc0000cdf0 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.582+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc081a19d0 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.584+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc08108070 con 0x7efc08105bf0 2024-04-06T14:12:08.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.587+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7efc00003a00 con 0x7efc08105bf0 2024-04-06T14:12:08.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.588+0000 7efbeffff700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 0x7efbf00a7ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:08.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.588+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7efc000ccb60 con 0x7efc08105bf0 2024-04-06T14:12:08.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.589+0000 7efc06d9d700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 0x7efbf00a7ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:08.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.591+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efc00095ff0 con 0x7efc08105bf0 2024-04-06T14:12:08.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.592+0000 7efc06d9d700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 0x7efbf00a7ba0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7efbf8000c00 tx=0x7efbf8005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:08.744 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:08 smithi090 ceph-mon[43420]: pgmap v206: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 46 MiB/s wr, 2.71k op/s 2024-04-06T14:12:08.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.986+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7efc081011e0 con 0x7efc08105bf0 2024-04-06T14:12:08.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.987+0000 7efbeffff700 1 -- 172.21.15.90:0/683498893 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7efc00099610 con 0x7efc08105bf0 2024-04-06T14:12:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:12:08.988 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:12:08.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.990+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 msgr2=0x7efbf00a7ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.990+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 0x7efbf00a7ba0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7efbf8000c00 tx=0x7efbf8005fb0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.990+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 msgr2=0x7efc08199720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:08.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.990+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 0x7efc08199720 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7efc0000efc0 tx=0x7efc0000c3a0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 shutdown_connections 2024-04-06T14:12:08.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efbf00a56e0 0x7efbf00a7ba0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efc08105bf0 0x7efc08199720 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efc080ffb30 0x7efc08194e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 --2- 172.21.15.90:0/683498893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efc081004f0 0x7efc08195380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 >> 172.21.15.90:0/683498893 conn(0x7efc080fb5e0 msgr2=0x7efc080fcd40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 shutdown_connections 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:08.991+0000 7efc0d0d1700 1 -- 172.21.15.90:0/683498893 wait complete. 2024-04-06T14:12:08.992 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:12:08.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:08 smithi105 ceph-mon[46969]: pgmap v206: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 46 MiB/s wr, 2.71k op/s 2024-04-06T14:12:09.010 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:09.011 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:12:09.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:08 smithi134 ceph-mon[47549]: pgmap v206: 97 pgs: 97 active+clean; 8.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 46 MiB/s wr, 2.71k op/s 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 -- 172.21.15.90:0/894783456 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f34480ff5e0 msgr2=0x7f34480ff9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 --2- 172.21.15.90:0/894783456 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f34480ff5e0 0x7f34480ff9c0 secure :-1 s=READY pgs=606 cs=0 l=1 rev1=1 crypto rx=0x7f344400a9a0 tx=0x7f344400acb0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 -- 172.21.15.90:0/894783456 shutdown_connections 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 --2- 172.21.15.90:0/894783456 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344810c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 --2- 172.21.15.90:0/894783456 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f34480ff5e0 0x7f34480ff9c0 unknown :-1 s=CLOSED pgs=606 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 --2- 172.21.15.90:0/894783456 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480fff90 0x7f3448103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 -- 172.21.15.90:0/894783456 >> 172.21.15.90:0/894783456 conn(0x7f34480761b0 msgr2=0x7f34480fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 -- 172.21.15.90:0/894783456 shutdown_connections 2024-04-06T14:12:09.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.203+0000 7f344f94d700 1 -- 172.21.15.90:0/894783456 wait complete. 2024-04-06T14:12:09.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.204+0000 7f344f94d700 1 Processor -- start 2024-04-06T14:12:09.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.204+0000 7f344f94d700 1 -- start start 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 0x7f344819bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344819c330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 0x7f34481953c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3448195b90 con 0x7f3448194f40 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3448195d00 con 0x7f34480ff5e0 2024-04-06T14:12:09.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.205+0000 7f344f94d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3448199940 con 0x7f3448104700 2024-04-06T14:12:09.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344cee8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344819c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344cee8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344819c330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42546/0 (socket says 172.21.15.90:42546) 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344cee8700 1 -- 172.21.15.90:0/116940571 learned_addr learned my addr 172.21.15.90:0/116940571 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 0x7f344819bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344deea700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 0x7f34481953c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 -- 172.21.15.90:0/116940571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 msgr2=0x7f344819c330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344819c330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 -- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 msgr2=0x7f34481953c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 0x7f34481953c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344d6e9700 1 -- 172.21.15.90:0/116940571 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f344400a610 con 0x7f34480ff5e0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.206+0000 7f344deea700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 0x7f34481953c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.207+0000 7f344d6e9700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 0x7f344819bdf0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f3444000c00 tx=0x7f344400d6d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.207+0000 7f343e7fc700 1 -- 172.21.15.90:0/116940571 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f344400d970 con 0x7f34480ff5e0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.207+0000 7f343e7fc700 1 -- 172.21.15.90:0/116940571 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3444015420 con 0x7f34480ff5e0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.207+0000 7f343e7fc700 1 -- 172.21.15.90:0/116940571 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f344401f670 con 0x7f34480ff5e0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.208+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3448199bc0 con 0x7f34480ff5e0 2024-04-06T14:12:09.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.208+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f344819a020 con 0x7f34480ff5e0 2024-04-06T14:12:09.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.212+0000 7f343e7fc700 1 -- 172.21.15.90:0/116940571 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f344400dad0 con 0x7f34480ff5e0 2024-04-06T14:12:09.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.212+0000 7f345bb01b80 1 Processor -- start 2024-04-06T14:12:09.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f343e7fc700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 0x7f34340a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 -- start start 2024-04-06T14:12:09.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9d2b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f343e7fc700 1 -- 172.21.15.90:0/116940571 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f3444011070 con 0x7f34480ff5e0 2024-04-06T14:12:09.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 0x5617dd9dc8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f344deea700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 0x7f34340a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9df1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd92a990 con 0x5617dd9dcde0 2024-04-06T14:12:09.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd928350 con 0x5617dd9d30a0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f345bb01b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd923c40 con 0x5617dd9d2780 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9d2b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9d2b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42550/0 (socket says 172.21.15.90:42550) 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f344d6e9700 1 -- 172.21.15.90:0/2234957202 learned_addr learned my addr 172.21.15.90:0/2234957202 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.213+0000 7f344cee8700 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 0x5617dd9dc8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 -- 172.21.15.90:0/2234957202 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 msgr2=0x5617dd9d2b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9d2b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 -- 172.21.15.90:0/2234957202 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 msgr2=0x5617dd9df1d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9df1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 -- 172.21.15.90:0/2234957202 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5617dd7a53b0 con 0x5617dd9d30a0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344cee8700 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 0x5617dd9dc8a0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f3448196630 tx=0x7f343800b470 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8278d4950bb80372 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f344deea700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 0x7f34340a37d0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x5617dd9d41f0 tx=0x7f34400093b0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f34337fe700 1 -- 172.21.15.90:0/2234957202 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3438010040 con 0x5617dd9d30a0 2024-04-06T14:12:09.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f34337fe700 1 -- 172.21.15.90:0/2234957202 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3438003d40 con 0x5617dd9d30a0 2024-04-06T14:12:09.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.214+0000 7f34337fe700 1 -- 172.21.15.90:0/2234957202 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3438016e40 con 0x5617dd9d30a0 2024-04-06T14:12:09.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 -- 172.21.15.90:0/2234957202 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 msgr2=0x5617dd9dc8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 0x5617dd9dc8a0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f3448196630 tx=0x7f343800b470 comp rx=0 tx=0).stop 2024-04-06T14:12:09.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 -- 172.21.15.90:0/2234957202 shutdown_connections 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9df1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9d2b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 --2- 172.21.15.90:0/2234957202 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d30a0 0x5617dd9dc8a0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 -- 172.21.15.90:0/2234957202 >> 172.21.15.90:0/2234957202 conn(0x5617dd9316a0 msgr2=0x5617dd9cda80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 -- 172.21.15.90:0/2234957202 shutdown_connections 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.215+0000 7f345bb01b80 1 -- 172.21.15.90:0/2234957202 wait complete. 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 -- start start 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 Processor -- start 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9dc850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9dcde0 0x5617dd9da280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd8bfd80 con 0x5617dd9d2780 2024-04-06T14:12:09.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd8bfa20 con 0x5617dd9dcde0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd7a76a0 con 0x5617dd9d30a0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f345bb01b80 10 client.0 _ll_register_callbacks cb 0x5617dd8c9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f344cee8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f344cee8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42566/0 (socket says 172.21.15.90:42566) 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.216+0000 7f344cee8700 1 -- 172.21.15.90:0/950270102 learned_addr learned my addr 172.21.15.90:0/950270102 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344deea700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9dcde0 0x5617dd9da280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344d6e9700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9dc850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 -- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9dcde0 msgr2=0x5617dd9da280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9dcde0 0x5617dd9da280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 -- 172.21.15.90:0/950270102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 msgr2=0x5617dd9dc850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9dc850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 -- 172.21.15.90:0/950270102 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f343800bac0 con 0x5617dd9d30a0 2024-04-06T14:12:09.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f344cee8700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f3438017d80 tx=0x7f3438017db0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=99945f4c51aa1ab1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f3430ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3438010070 con 0x5617dd9d30a0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5617dd8e4380 con 0x5617dd9d30a0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f3438005ce0 con 0x5617dd9d30a0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3438003880 con 0x5617dd9d30a0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f345bb01b80 10 client.26119 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.217+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5617dd751b60 con 0x5617dd9d30a0 2024-04-06T14:12:09.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.218+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5617dd8e4380 con 0x5617dd9d30a0 2024-04-06T14:12:09.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3438005040 con 0x5617dd9d30a0 2024-04-06T14:12:09.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 10 client.26119 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 10 client.26119 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 20 client.26119 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dd9fd970 0x5617dd9ffd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 4 client.26119 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5617dd91c700 con 0x5617dd9fd970 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.219+0000 7f344d6e9700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dd9fd970 0x5617dd9ffd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.220+0000 7f344d6e9700 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dd9fd970 0x5617dd9ffd90 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d59604fb01ed4d21 server_cookie=cb0ffeaee1f68c5e in_seq=0 out_seq=0 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.220+0000 7f3430ff9700 10 client.26119 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f3430ff9700 1 -- 172.21.15.90:0/950270102 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5617dd91c700 con 0x5617dd9fd970 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f3430ff9700 10 client.26119 handle_command_reply: tid=0 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 client.26119 shutdown 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 msgr2=0x5617dd9d9b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f3438017d80 tx=0x7f3438017db0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 shutdown_connections 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dd9fd970 0x5617dd9ffd90 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9dcde0 0x5617dd9da280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9dc850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 --2- 172.21.15.90:0/950270102 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d30a0 0x5617dd9d9b90 secure :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f3438017d80 tx=0x7f3438017db0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 >> 172.21.15.90:0/950270102 conn(0x5617dd9316a0 msgr2=0x5617dd9ccc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.221+0000 7f3430ff9700 0 client.26119 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.222+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 shutdown_connections 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.222+0000 7f345bb01b80 1 -- 172.21.15.90:0/950270102 wait complete. 2024-04-06T14:12:09.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.222+0000 7f345bb01b80 20 client.26119 trim_cache size 0 max 16384 2024-04-06T14:12:09.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.238+0000 7f345bb01b80 1 Processor -- start 2024-04-06T14:12:09.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 -- start start 2024-04-06T14:12:09.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9dcde0 0x5617dd9e3a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9d88c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd9db170 con 0x5617dd9d2780 2024-04-06T14:12:09.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd9d34b0 con 0x5617dd9d8460 2024-04-06T14:12:09.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f345bb01b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd9d3a60 con 0x5617dd9dcde0 2024-04-06T14:12:09.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59174/0 (socket says 172.21.15.90:59174) 2024-04-06T14:12:09.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.239+0000 7f344d6e9700 1 -- 172.21.15.90:0/3803511159 learned_addr learned my addr 172.21.15.90:0/3803511159 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344cee8700 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9dcde0 0x5617dd9e3a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344deea700 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9d88c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 -- 172.21.15.90:0/3803511159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9dcde0 msgr2=0x5617dd9e3a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9dcde0 0x5617dd9e3a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 -- 172.21.15.90:0/3803511159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 msgr2=0x5617dd9d88c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9d88c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 -- 172.21.15.90:0/3803511159 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5617dd8e9ea0 con 0x5617dd9d2780 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f34440ce410 tx=0x7f3444086040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa3a292c77e4682a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.240+0000 7f3433fff700 1 -- 172.21.15.90:0/3803511159 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f344409bc90 con 0x5617dd9d2780 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.241+0000 7f3433fff700 1 -- 172.21.15.90:0/3803511159 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f344409b590 con 0x5617dd9d2780 2024-04-06T14:12:09.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.241+0000 7f3433fff700 1 -- 172.21.15.90:0/3803511159 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f344400dd80 con 0x5617dd9d2780 2024-04-06T14:12:09.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.241+0000 7f345bb01b80 1 -- 172.21.15.90:0/3803511159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 msgr2=0x5617dd9ffd60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.241+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 secure :-1 s=READY pgs=607 cs=0 l=1 rev1=1 crypto rx=0x7f34440ce410 tx=0x7f3444086040 comp rx=0 tx=0).stop 2024-04-06T14:12:09.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.242+0000 7f345bb01b80 1 -- 172.21.15.90:0/3803511159 shutdown_connections 2024-04-06T14:12:09.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.242+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9d88c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.242+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9d2780 0x5617dd9ffd60 unknown :-1 s=CLOSED pgs=607 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.242+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3803511159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9dcde0 0x5617dd9e3a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.242+0000 7f345bb01b80 1 -- 172.21.15.90:0/3803511159 >> 172.21.15.90:0/3803511159 conn(0x5617dd9316a0 msgr2=0x5617dda046f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.243+0000 7f345bb01b80 1 -- 172.21.15.90:0/3803511159 shutdown_connections 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.243+0000 7f345bb01b80 1 -- 172.21.15.90:0/3803511159 wait complete. 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.243+0000 7f345bb01b80 1 -- start start 2024-04-06T14:12:09.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.243+0000 7f345bb01b80 1 Processor -- start 2024-04-06T14:12:09.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9e1e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9e23b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd9e2a90 con 0x5617dd9dcde0 2024-04-06T14:12:09.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dd9e2c00 con 0x5617dd9d8460 2024-04-06T14:12:09.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5617dda517b0 con 0x5617dd9d2780 2024-04-06T14:12:09.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f345bb01b80 10 client.0 _ll_register_callbacks cb 0x5617dd8c9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42584/0 (socket says 172.21.15.90:42584) 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 -- 172.21.15.90:0/3176905715 learned_addr learned my addr 172.21.15.90:0/3176905715 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344deea700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9e23b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344cee8700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9e1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 -- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 msgr2=0x5617dd9e1e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9e1e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 -- 172.21.15.90:0/3176905715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 msgr2=0x5617dd9e23b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9e23b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.244+0000 7f344d6e9700 1 -- 172.21.15.90:0/3176905715 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f344819a020 con 0x5617dd9d2780 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f344deea700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9e23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f344cee8700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9e1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f344d6e9700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f34440cbbf0 tx=0x7f344409bd40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8819bf9f848259c1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f3430ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f344409f030 con 0x5617dd9d2780 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f344409f190 con 0x5617dd9d2780 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3438003d40 con 0x5617dd9d2780 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.245+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f34440cbdc0 con 0x5617dd9d2780 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 10 client.26125 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5617dd7a9cd0 con 0x5617dd9d2780 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3438003d40 con 0x5617dd9d2780 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f34440a34f0 con 0x5617dd9d2780 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 10 client.26125 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 10 client.26125 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 20 client.26125 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dda528e0 0x5617dda54cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 4 client.26125 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:09.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.246+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5617dd772cf0 con 0x5617dda528e0 2024-04-06T14:12:09.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.247+0000 7f344cee8700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dda528e0 0x5617dda54cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.247+0000 7f3430ff9700 10 client.26125 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:09.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.247+0000 7f344cee8700 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dda528e0 0x5617dda54cd0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a362e0d553a4b15a server_cookie=4bcb38ad66c07eff in_seq=0 out_seq=0 2024-04-06T14:12:09.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f3430ff9700 1 -- 172.21.15.90:0/3176905715 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5617dd772cf0 con 0x5617dda528e0 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f3430ff9700 10 client.26125 handle_command_reply: tid=0 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 client.26125 shutdown 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 msgr2=0x5617dd9e39c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f34440cbbf0 tx=0x7f344409bd40 comp rx=0 tx=0).stop 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 shutdown_connections 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5617dda528e0 0x5617dda54cd0 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5617dd9dcde0 0x5617dd9e23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5617dd9d2780 0x5617dd9e39c0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 --2- 172.21.15.90:0/3176905715 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5617dd9d8460 0x5617dd9e1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f3430ff9700 0 client.26125 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.248+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 >> 172.21.15.90:0/3176905715 conn(0x5617dd9316a0 msgr2=0x5617dd9d88a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 shutdown_connections 2024-04-06T14:12:09.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f345bb01b80 1 -- 172.21.15.90:0/3176905715 wait complete. 2024-04-06T14:12:09.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f345bb01b80 20 client.26125 trim_cache size 0 max 16384 2024-04-06T14:12:09.258 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:09.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 msgr2=0x7f34340a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 0x7f34340a37d0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x5617dd9d41f0 tx=0x7f34400093b0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 msgr2=0x7f344819bdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.249+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 0x7f344819bdf0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f3444000c00 tx=0x7f344400d6d0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 shutdown_connections 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34340a1310 0x7f34340a37d0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3448194f40 0x7f34481953c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34480ff5e0 0x7f344819bdf0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 --2- 172.21.15.90:0/116940571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3448104700 0x7f344819c330 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 >> 172.21.15.90:0/116940571 conn(0x7f34480761b0 msgr2=0x7f3448102810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 shutdown_connections 2024-04-06T14:12:09.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.250+0000 7f344f94d700 1 -- 172.21.15.90:0/116940571 wait complete. 2024-04-06T14:12:09.268 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:09.269 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:12:09.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.457+0000 7f4783165700 1 -- 172.21.15.90:0/2110456241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c067e60 msgr2=0x7f477c068240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.457+0000 7f4783165700 1 --2- 172.21.15.90:0/2110456241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c067e60 0x7f477c068240 secure :-1 s=READY pgs=608 cs=0 l=1 rev1=1 crypto rx=0x7f4770009990 tx=0x7f4770009ca0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 -- 172.21.15.90:0/2110456241 shutdown_connections 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 --2- 172.21.15.90:0/2110456241 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c10d750 0x7f477c10fb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 --2- 172.21.15.90:0/2110456241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c067e60 0x7f477c068240 unknown :-1 s=CLOSED pgs=608 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 --2- 172.21.15.90:0/2110456241 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c068780 0x7f477c10d210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 -- 172.21.15.90:0/2110456241 >> 172.21.15.90:0/2110456241 conn(0x7f477c076d50 msgr2=0x7f477c077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 -- 172.21.15.90:0/2110456241 shutdown_connections 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.458+0000 7f4783165700 1 -- 172.21.15.90:0/2110456241 wait complete. 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 Processor -- start 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 -- start start 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c068780 0x7f477c199900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 0x7f477c19dca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f477c19e2e0 con 0x7f477c068780 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f477c19e450 con 0x7f477c067e60 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4783165700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f477c19e610 con 0x7f477c10d750 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4780f01700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4780f01700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54000/0 (socket says 172.21.15.90:54000) 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f4780f01700 1 -- 172.21.15.90:0/1452981659 learned_addr learned my addr 172.21.15.90:0/1452981659 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.459+0000 7f477bfff700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c068780 0x7f477c199900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4781702700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 0x7f477c19dca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 -- 172.21.15.90:0/1452981659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 msgr2=0x7f477c19dca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 0x7f477c19dca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 -- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c068780 msgr2=0x7f477c199900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c068780 0x7f477c199900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 -- 172.21.15.90:0/1452981659 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47700095f0 con 0x7f477c067e60 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4781702700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 0x7f477c19dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:09.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4780f01700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4770000c00 tx=0x7f4770004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4779ffb700 1 -- 172.21.15.90:0/1452981659 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f477001d070 con 0x7f477c067e60 2024-04-06T14:12:09.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.460+0000 7f4779ffb700 1 -- 172.21.15.90:0/1452981659 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4770022470 con 0x7f477c067e60 2024-04-06T14:12:09.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.461+0000 7f4779ffb700 1 -- 172.21.15.90:0/1452981659 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4770017650 con 0x7f477c067e60 2024-04-06T14:12:09.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.461+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f477c19e890 con 0x7f477c067e60 2024-04-06T14:12:09.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.461+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f477c19eb10 con 0x7f477c067e60 2024-04-06T14:12:09.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.464+0000 7f4779ffb700 1 -- 172.21.15.90:0/1452981659 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f47700225e0 con 0x7f477c067e60 2024-04-06T14:12:09.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.466+0000 7f4779ffb700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 0x7f476c0a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.466+0000 7f4779ffb700 1 -- 172.21.15.90:0/1452981659 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f47700cdd20 con 0x7f477c067e60 2024-04-06T14:12:09.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.466+0000 7f478f319b80 1 Processor -- start 2024-04-06T14:12:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.473+0000 7f477bfff700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 0x7f476c0a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 -- start start 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c63269d370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a99e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6325ec990 con 0x55c63269d8b0 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6325ea350 con 0x55c6326a75f0 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f478f319b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6325e5c40 con 0x55c63269cf90 2024-04-06T14:12:09.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f477bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.474+0000 7f477bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59204/0 (socket says 172.21.15.90:59204) 2024-04-06T14:12:09.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 -- 172.21.15.90:0/1771721562 learned_addr learned my addr 172.21.15.90:0/1771721562 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 -- 172.21.15.90:0/1771721562 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 msgr2=0x55c63269d370 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:09.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c63269d370 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 -- 172.21.15.90:0/1771721562 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 msgr2=0x55c6326a99e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a99e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.475+0000 7f477bfff700 1 -- 172.21.15.90:0/1771721562 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c6324673b0 con 0x55c63269d8b0 2024-04-06T14:12:09.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.476+0000 7f477bfff700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 0x7f476c0a38a0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f47680126c0 tx=0x7f476800bc00 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.476+0000 7f477bfff700 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f4768012e10 tx=0x7f476801a000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d36f3719875b633d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.481+0000 7f4766ffd700 1 -- 172.21.15.90:0/1771721562 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4768012030 con 0x55c63269d8b0 2024-04-06T14:12:09.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.481+0000 7f4766ffd700 1 -- 172.21.15.90:0/1771721562 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4768012190 con 0x55c63269d8b0 2024-04-06T14:12:09.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.481+0000 7f4766ffd700 1 -- 172.21.15.90:0/1771721562 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f476801f450 con 0x55c63269d8b0 2024-04-06T14:12:09.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.485+0000 7f478f319b80 1 -- 172.21.15.90:0/1771721562 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 msgr2=0x55c6326a70b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.485+0000 7f478f319b80 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 secure :-1 s=READY pgs=609 cs=0 l=1 rev1=1 crypto rx=0x7f4768012e10 tx=0x7f476801a000 comp rx=0 tx=0).stop 2024-04-06T14:12:09.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.487+0000 7f478f319b80 1 -- 172.21.15.90:0/1771721562 shutdown_connections 2024-04-06T14:12:09.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.487+0000 7f478f319b80 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a99e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.487+0000 7f478f319b80 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c63269d370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.487+0000 7f478f319b80 1 --2- 172.21.15.90:0/1771721562 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a70b0 unknown :-1 s=CLOSED pgs=609 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.487+0000 7f478f319b80 1 -- 172.21.15.90:0/1771721562 >> 172.21.15.90:0/1771721562 conn(0x55c6325f36a0 msgr2=0x55c63268fa40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.490+0000 7f478f319b80 1 -- 172.21.15.90:0/1771721562 shutdown_connections 2024-04-06T14:12:09.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.502+0000 7f478f319b80 1 -- 172.21.15.90:0/1771721562 wait complete. 2024-04-06T14:12:09.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.502+0000 7f478f319b80 1 -- start start 2024-04-06T14:12:09.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.502+0000 7f478f319b80 1 Processor -- start 2024-04-06T14:12:09.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326a4420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c63269d8b0 0x55c6326a4960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c632581d80 con 0x55c63269cf90 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c632581a20 con 0x55c63269d8b0 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6324696a0 con 0x55c6326a75f0 2024-04-06T14:12:09.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f478f319b80 10 client.0 _ll_register_callbacks cb 0x55c63258b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f477bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c63269d8b0 0x55c6326a4960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f4781702700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f4781702700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42620/0 (socket says 172.21.15.90:42620) 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f4781702700 1 -- 172.21.15.90:0/126282193 learned_addr learned my addr 172.21.15.90:0/126282193 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.503+0000 7f4780f01700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326a4420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4781702700 1 -- 172.21.15.90:0/126282193 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c63269d8b0 msgr2=0x55c6326a4960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4781702700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c63269d8b0 0x55c6326a4960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4781702700 1 -- 172.21.15.90:0/126282193 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 msgr2=0x55c6326a4420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4781702700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326a4420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4781702700 1 -- 172.21.15.90:0/126282193 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f476800e410 con 0x55c6326a75f0 2024-04-06T14:12:09.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.504+0000 7f4780f01700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326a4420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:09.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.523+0000 7f4781702700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f4774007af0 tx=0x7f4774007eb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8cfe836bfed49d63 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.523+0000 7f475bfff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:12:09.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.536+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f477400f900 con 0x55c6326a75f0 2024-04-06T14:12:09.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.536+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f477400fa60 con 0x55c6326a75f0 2024-04-06T14:12:09.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.536+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c6325a6380 con 0x55c6326a75f0 2024-04-06T14:12:09.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.543+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f477400fc20 con 0x55c6326a75f0 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.543+0000 7f478f319b80 10 client.26131 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.543+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c632413b60 con 0x55c6326a75f0 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.544+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c6325a6380 con 0x55c6326a75f0 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4774014070 con 0x55c6326a75f0 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 10 client.26131 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 10 client.26131 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:12:09.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 20 client.26131 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c6326c82c0 0x55c6326ca6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 4 client.26131 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:09.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c6325de700 con 0x55c6326c82c0 2024-04-06T14:12:09.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.545+0000 7f4780f01700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c6326c82c0 0x55c6326ca6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.553+0000 7f4780f01700 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c6326c82c0 0x55c6326ca6e0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=1321bb544a9cb934 server_cookie=224ac469ac1f6978 in_seq=0 out_seq=0 2024-04-06T14:12:09.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.553+0000 7f475bfff700 10 client.26131 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:09.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.555+0000 7f475bfff700 1 -- 172.21.15.90:0/126282193 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c6325de700 con 0x55c6326c82c0 2024-04-06T14:12:09.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.555+0000 7f475bfff700 10 client.26131 handle_command_reply: tid=0 2024-04-06T14:12:09.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.555+0000 7f478f319b80 1 client.26131 shutdown 2024-04-06T14:12:09.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.555+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 msgr2=0x55c6326a4ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.555+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f4774007af0 tx=0x7f4774007eb0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 shutdown_connections 2024-04-06T14:12:09.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c6326c82c0 0x55c6326ca6e0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6326a75f0 0x55c6326a4ea0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326a4420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 --2- 172.21.15.90:0/126282193 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c63269d8b0 0x55c6326a4960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 >> 172.21.15.90:0/126282193 conn(0x55c6325f36a0 msgr2=0x55c63268ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f475bfff700 0 client.26131 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:09.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 shutdown_connections 2024-04-06T14:12:09.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 1 -- 172.21.15.90:0/126282193 wait complete. 2024-04-06T14:12:09.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.556+0000 7f478f319b80 20 client.26131 trim_cache size 0 max 16384 2024-04-06T14:12:09.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.574+0000 7f478f319b80 1 Processor -- start 2024-04-06T14:12:09.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 -- start start 2024-04-06T14:12:09.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c6326a2cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a1680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c63268f3e0 con 0x55c63269d8b0 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6326a73a0 con 0x55c6326a75f0 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f478f319b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6326a5a20 con 0x55c63269cf90 2024-04-06T14:12:09.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f477bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f477bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59212/0 (socket says 172.21.15.90:59212) 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.575+0000 7f477bfff700 1 -- 172.21.15.90:0/802418106 learned_addr learned my addr 172.21.15.90:0/802418106 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f4780f01700 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c6326a2cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 -- 172.21.15.90:0/802418106 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 msgr2=0x55c6326a2cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c6326a2cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 -- 172.21.15.90:0/802418106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 msgr2=0x55c6326a1680 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a1680 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 -- 172.21.15.90:0/802418106 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c6325abea0 con 0x55c63269d8b0 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f477bfff700 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f4768022040 tx=0x7f4768005ce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ce1a51400b32a066 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f47677fe700 1 -- 172.21.15.90:0/802418106 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f476800b8e0 con 0x55c63269d8b0 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f47677fe700 1 -- 172.21.15.90:0/802418106 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4768010920 con 0x55c63269d8b0 2024-04-06T14:12:09.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.576+0000 7f47677fe700 1 -- 172.21.15.90:0/802418106 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4768005760 con 0x55c63269d8b0 2024-04-06T14:12:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 -- 172.21.15.90:0/802418106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 msgr2=0x55c6326a3230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 secure :-1 s=READY pgs=610 cs=0 l=1 rev1=1 crypto rx=0x7f4768022040 tx=0x7f4768005ce0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 -- 172.21.15.90:0/802418106 shutdown_connections 2024-04-06T14:12:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326a1680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269cf90 0x55c6326a2cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 --2- 172.21.15.90:0/802418106 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269d8b0 0x55c6326a3230 unknown :-1 s=CLOSED pgs=610 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 -- 172.21.15.90:0/802418106 >> 172.21.15.90:0/802418106 conn(0x55c6325f36a0 msgr2=0x55c63268f570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 -- 172.21.15.90:0/802418106 shutdown_connections 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.577+0000 7f478f319b80 1 -- 172.21.15.90:0/802418106 wait complete. 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 -- start start 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 Processor -- start 2024-04-06T14:12:09.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269d8b0 0x55c6326ac0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326ac610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c63271b2a0 con 0x55c63269cf90 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c63271b410 con 0x55c6326a75f0 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c63271b5d0 con 0x55c63269d8b0 2024-04-06T14:12:09.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.578+0000 7f478f319b80 10 client.0 _ll_register_callbacks cb 0x55c63258b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4781702700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326ac610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59216/0 (socket says 172.21.15.90:59216) 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 -- 172.21.15.90:0/17282264 learned_addr learned my addr 172.21.15.90:0/17282264 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f477bfff700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269d8b0 0x55c6326ac0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 -- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269d8b0 msgr2=0x55c6326ac0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269d8b0 0x55c6326ac0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 -- 172.21.15.90:0/17282264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 msgr2=0x55c6326ac610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326ac610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.579+0000 7f4780f01700 1 -- 172.21.15.90:0/17282264 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c6324673b0 con 0x55c63269cf90 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f4780f01700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f477009ae80 tx=0x7f4770009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=600f23a783a0afb2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f475bfff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f477000fcc0 con 0x55c63269cf90 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f477009aef0 con 0x55c63269cf90 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4768012190 con 0x55c63269cf90 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f477009ed20 con 0x55c63269cf90 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f478f319b80 10 client.16545 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c63246bcd0 con 0x55c63269cf90 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.580+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4768012190 con 0x55c63269cf90 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f477009a4e0 con 0x55c63269cf90 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 10 client.16545 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 10 client.16545 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 20 client.16545 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c63271ca50 0x55c63271ee40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 4 client.16545 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c632434cf0 con 0x55c63271ca50 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f477bfff700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c63271ca50 0x55c63271ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f475bfff700 10 client.16545 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f477bfff700 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c63271ca50 0x55c63271ee40 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e830b31c09be158 server_cookie=7f62d4c66574dfca in_seq=0 out_seq=0 2024-04-06T14:12:09.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f475bfff700 1 -- 172.21.15.90:0/17282264 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c632434cf0 con 0x55c63271ca50 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f475bfff700 10 client.16545 handle_command_reply: tid=0 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 1 client.16545 shutdown 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 msgr2=0x55c6326abb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.581+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 secure :-1 s=READY pgs=611 cs=0 l=1 rev1=1 crypto rx=0x7f477009ae80 tx=0x7f4770009310 comp rx=0 tx=0).stop 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 shutdown_connections 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c63271ca50 0x55c63271ee40 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6326a75f0 0x55c6326ac610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c63269cf90 0x55c6326abb70 unknown :-1 s=CLOSED pgs=611 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 --2- 172.21.15.90:0/17282264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c63269d8b0 0x55c6326ac0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 >> 172.21.15.90:0/17282264 conn(0x55c6325f36a0 msgr2=0x55c6326a2c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f475bfff700 0 client.16545 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 shutdown_connections 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 1 -- 172.21.15.90:0/17282264 wait complete. 2024-04-06T14:12:09.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f478f319b80 20 client.16545 trim_cache size 0 max 16384 2024-04-06T14:12:09.598 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:09.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 msgr2=0x7f476c0a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 0x7f476c0a38a0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f47680126c0 tx=0x7f476800bc00 comp rx=0 tx=0).stop 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 msgr2=0x7f477c1993c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4770000c00 tx=0x7f4770004730 comp rx=0 tx=0).stop 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 shutdown_connections 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f476c0a13e0 0x7f476c0a38a0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f477c10d750 0x7f477c19dca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f477c067e60 0x7f477c1993c0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 --2- 172.21.15.90:0/1452981659 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f477c068780 0x7f477c199900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.582+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 >> 172.21.15.90:0/1452981659 conn(0x7f477c076d50 msgr2=0x7f477c10ed10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.583+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 shutdown_connections 2024-04-06T14:12:09.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.583+0000 7f4783165700 1 -- 172.21.15.90:0/1452981659 wait complete. 2024-04-06T14:12:09.602 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:09.605 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:12:09.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 -- 172.21.15.90:0/80771651 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3460100820 msgr2=0x7f3460104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 --2- 172.21.15.90:0/80771651 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3460100820 0x7f3460104840 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f34540098f0 tx=0x7f3454009c00 comp rx=0 tx=0).stop 2024-04-06T14:12:09.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 -- 172.21.15.90:0/80771651 shutdown_connections 2024-04-06T14:12:09.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 --2- 172.21.15.90:0/80771651 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f346010caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 --2- 172.21.15.90:0/80771651 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f34600ffe70 0x7f3460100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 --2- 172.21.15.90:0/80771651 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3460100820 0x7f3460104840 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.794+0000 7f346526c700 1 -- 172.21.15.90:0/80771651 >> 172.21.15.90:0/80771651 conn(0x7f3460076d50 msgr2=0x7f3460077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.795+0000 7f346526c700 1 -- 172.21.15.90:0/80771651 shutdown_connections 2024-04-06T14:12:09.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.795+0000 7f346526c700 1 -- 172.21.15.90:0/80771651 wait complete. 2024-04-06T14:12:09.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.796+0000 7f346526c700 1 Processor -- start 2024-04-06T14:12:09.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.796+0000 7f346526c700 1 -- start start 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 0x7f3460109560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f34601a3680 0x7f34601a5a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34601a6080 con 0x7f3460104f70 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34601a61f0 con 0x7f34601090e0 2024-04-06T14:12:09.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f346526c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34601a63b0 con 0x7f34601a3680 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 0x7f3460109560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59234/0 (socket says 172.21.15.90:59234) 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 -- 172.21.15.90:0/3960921588 learned_addr learned my addr 172.21.15.90:0/3960921588 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 -- 172.21.15.90:0/3960921588 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f34601a3680 msgr2=0x7f34601a5a70 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f34601a3680 0x7f34601a5a70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 -- 172.21.15.90:0/3960921588 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 msgr2=0x7f3460109560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 0x7f3460109560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345effd700 1 -- 172.21.15.90:0/3960921588 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3454009520 con 0x7f3460104f70 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.797+0000 7f345e7fc700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 0x7f3460109560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f345effd700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f345000b480 tx=0x7f345000b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f3447fff700 1 -- 172.21.15.90:0/3960921588 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3450011830 con 0x7f3460104f70 2024-04-06T14:12:09.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f3447fff700 1 -- 172.21.15.90:0/3960921588 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3450011e50 con 0x7f3460104f70 2024-04-06T14:12:09.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f34601a6690 con 0x7f3460104f70 2024-04-06T14:12:09.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f3447fff700 1 -- 172.21.15.90:0/3960921588 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345000f2e0 con 0x7f3460104f70 2024-04-06T14:12:09.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.798+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f34601a6b50 con 0x7f3460104f70 2024-04-06T14:12:09.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.801+0000 7f3447fff700 1 -- 172.21.15.90:0/3960921588 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3450010580 con 0x7f3460104f70 2024-04-06T14:12:09.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.802+0000 7f3471420b80 1 Processor -- start 2024-04-06T14:12:09.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.802+0000 7f3447fff700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 0x7f34480a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.802+0000 7f3447fff700 1 -- 172.21.15.90:0/3960921588 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f34500cc650 con 0x7f3460104f70 2024-04-06T14:12:09.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 -- start start 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e4f3560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4f8390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e442990 con 0x55962e4f3180 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e440350 con 0x55962e4f3aa0 2024-04-06T14:12:09.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.803+0000 7f3471420b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e43bc40 con 0x55962e4f7f30 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e4f3560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54044/0 (socket says 172.21.15.90:54044) 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 -- 172.21.15.90:0/4096497950 learned_addr learned my addr 172.21.15.90:0/4096497950 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 -- 172.21.15.90:0/4096497950 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 msgr2=0x55962e4f8390 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4f8390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 -- 172.21.15.90:0/4096497950 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 msgr2=0x55962e4f3560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e4f3560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 -- 172.21.15.90:0/4096497950 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55962e2bd3b0 con 0x55962e4f3aa0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f345e7fc700 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f34540098f0 tx=0x7f345400bef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1f65d470af259d86 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f34457fa700 1 -- 172.21.15.90:0/4096497950 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345401c050 con 0x55962e4f3aa0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f34457fa700 1 -- 172.21.15.90:0/4096497950 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f345401d040 con 0x55962e4f3aa0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.804+0000 7f34457fa700 1 -- 172.21.15.90:0/4096497950 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f34540053f0 con 0x55962e4f3aa0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f345f7fe700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 0x7f34480a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 -- 172.21.15.90:0/4096497950 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 msgr2=0x55962e4f79f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f34540098f0 tx=0x7f345400bef0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 -- 172.21.15.90:0/4096497950 shutdown_connections 2024-04-06T14:12:09.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4f8390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e4f3560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 --2- 172.21.15.90:0/4096497950 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3aa0 0x55962e4f79f0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 -- 172.21.15.90:0/4096497950 >> 172.21.15.90:0/4096497950 conn(0x55962e4496a0 msgr2=0x55962e4e5a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 -- 172.21.15.90:0/4096497950 shutdown_connections 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.805+0000 7f3471420b80 1 -- 172.21.15.90:0/4096497950 wait complete. 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 -- start start 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 Processor -- start 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e4ff710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4fd190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e3d7d80 con 0x55962e4f3aa0 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e3d7a20 con 0x55962e4f3180 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e2bf6a0 con 0x55962e4f7f30 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f3471420b80 10 client.0 _ll_register_callbacks cb 0x55962e3e1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59252/0 (socket says 172.21.15.90:59252) 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f345effd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e4ff710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.806+0000 7f345e7fc700 1 -- 172.21.15.90:0/2344635240 learned_addr learned my addr 172.21.15.90:0/2344635240 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345f7fe700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4fd190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 -- 172.21.15.90:0/2344635240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 msgr2=0x55962e4fd190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4fd190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 -- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 msgr2=0x55962e4ff710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e4ff710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 -- 172.21.15.90:0/2344635240 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f346010d360 con 0x55962e4f3aa0 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345f7fe700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4fd190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345f7fe700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 0x7f34480a3750 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f345800e420 tx=0x7f345800b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f345e7fc700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f3454000c00 tx=0x7f3454005380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9ff948fece95d31 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3442ff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345400b760 con 0x55962e4f3aa0 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55962e3fc380 con 0x55962e4f3aa0 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f345400b8c0 con 0x55962e4f3aa0 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f345401c020 con 0x55962e4f3aa0 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3471420b80 10 client.16557 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55962e269b60 con 0x55962e4f3aa0 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.807+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f345403b020 con 0x55962e4f3aa0 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.808+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3454009c70 con 0x55962e4f3aa0 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 10 client.16557 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 10 client.16557 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 20 client.16557 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e51dbf0 0x55962e51ffe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 4 client.16557 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55962e434700 con 0x55962e51dbf0 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f345effd700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e51dbf0 0x55962e51ffe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f3442ff5700 10 client.16557 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.809+0000 7f345effd700 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e51dbf0 0x55962e51ffe0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8731d09b922835ab server_cookie=1ec2df7b9cb16f0d in_seq=0 out_seq=0 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3442ff5700 1 -- 172.21.15.90:0/2344635240 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55962e434700 con 0x55962e51dbf0 2024-04-06T14:12:09.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3442ff5700 10 client.16557 handle_command_reply: tid=0 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 client.16557 shutdown 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 msgr2=0x55962e4fcaa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 secure :-1 s=READY pgs=613 cs=0 l=1 rev1=1 crypto rx=0x7f3454000c00 tx=0x7f3454005380 comp rx=0 tx=0).stop 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 shutdown_connections 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e51dbf0 0x55962e51ffe0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e4fd190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3442ff5700 0 client.16557 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:09.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e4ff710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 --2- 172.21.15.90:0/2344635240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e4fcaa0 unknown :-1 s=CLOSED pgs=613 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 >> 172.21.15.90:0/2344635240 conn(0x55962e4496a0 msgr2=0x55962e4f1f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 shutdown_connections 2024-04-06T14:12:09.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.810+0000 7f3471420b80 1 -- 172.21.15.90:0/2344635240 wait complete. 2024-04-06T14:12:09.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.811+0000 7f3471420b80 20 client.16557 trim_cache size 0 max 16384 2024-04-06T14:12:09.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 Processor -- start 2024-04-06T14:12:09.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 -- start start 2024-04-06T14:12:09.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e515100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e51ac20 con 0x55962e4f3180 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e4e6dc0 con 0x55962e4f7f30 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f3471420b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e4fbe40 con 0x55962e4f3aa0 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42654/0 (socket says 172.21.15.90:42654) 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345e7fc700 1 -- 172.21.15.90:0/2993540326 learned_addr learned my addr 172.21.15.90:0/2993540326 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54054/0 (socket says 172.21.15.90:54054) 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 -- 172.21.15.90:0/2993540326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 msgr2=0x55962e4fb390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 -- 172.21.15.90:0/2993540326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 msgr2=0x55962e515100 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:09.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e515100 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345f7fe700 1 -- 172.21.15.90:0/2993540326 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55962e401ea0 con 0x55962e4f7f30 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.827+0000 7f345e7fc700 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.828+0000 7f345f7fe700 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f3458009750 tx=0x7f3458015000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6273e3f5c3596f4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.828+0000 7f3445ffb700 1 -- 172.21.15.90:0/2993540326 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458023070 con 0x55962e4f7f30 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.828+0000 7f3445ffb700 1 -- 172.21.15.90:0/2993540326 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f345800f040 con 0x55962e4f7f30 2024-04-06T14:12:09.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.828+0000 7f3445ffb700 1 -- 172.21.15.90:0/2993540326 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3458004280 con 0x55962e4f7f30 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- 172.21.15.90:0/2993540326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 msgr2=0x55962e4fb8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f3458009750 tx=0x7f3458015000 comp rx=0 tx=0).stop 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- 172.21.15.90:0/2993540326 shutdown_connections 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f7f30 0x55962e4fb8d0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3180 0x55962e515100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 --2- 172.21.15.90:0/2993540326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f3aa0 0x55962e4fb390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- 172.21.15.90:0/2993540326 >> 172.21.15.90:0/2993540326 conn(0x55962e4496a0 msgr2=0x55962e4f6940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- 172.21.15.90:0/2993540326 shutdown_connections 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- 172.21.15.90:0/2993540326 wait complete. 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.829+0000 7f3471420b80 1 -- start start 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 Processor -- start 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e514e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e513a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e5140a0 con 0x55962e4f3aa0 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e514210 con 0x55962e4f3180 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55962e5143d0 con 0x55962e4f7f30 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3471420b80 10 client.0 _ll_register_callbacks cb 0x7f3450009380 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345effd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e514e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59266/0 (socket says 172.21.15.90:59266) 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 -- 172.21.15.90:0/423000528 learned_addr learned my addr 172.21.15.90:0/423000528 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345f7fe700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e513a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 -- 172.21.15.90:0/423000528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 msgr2=0x55962e513a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e513a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 -- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 msgr2=0x55962e514e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e514e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 -- 172.21.15.90:0/423000528 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55962e269b60 con 0x55962e4f3aa0 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345effd700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e514e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f345e7fc700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f3454009e30 tx=0x7f3454009e60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7f3eec4bfb1cfcbc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.830+0000 7f3442ff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f345401c070 con 0x55962e4f3aa0 2024-04-06T14:12:09.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f345401d040 con 0x55962e4f3aa0 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f345403c080 con 0x55962e4f3aa0 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f345400b970 con 0x55962e4f3aa0 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 10 client.16560 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f345800eda0 con 0x55962e4f3aa0 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f345404c080 con 0x55962e4f3aa0 2024-04-06T14:12:09.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3454046030 con 0x55962e4f3aa0 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 10 client.16560 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 10 client.16560 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 20 client.16560 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e572c00 0x55962e575020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 4 client.16560 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:09.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55962e28acf0 con 0x55962e572c00 2024-04-06T14:12:09.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.831+0000 7f345effd700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e572c00 0x55962e575020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:09.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3442ff5700 10 client.16560 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:09.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f345effd700 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e572c00 0x55962e575020 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7c4f69a09206023 server_cookie=9644987209d8cca2 in_seq=0 out_seq=0 2024-04-06T14:12:09.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3442ff5700 1 -- 172.21.15.90:0/423000528 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55962e28acf0 con 0x55962e572c00 2024-04-06T14:12:09.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3442ff5700 10 client.16560 handle_command_reply: tid=0 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 client.16560 shutdown 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 msgr2=0x55962e513510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 secure :-1 s=READY pgs=614 cs=0 l=1 rev1=1 crypto rx=0x7f3454009e30 tx=0x7f3454009e60 comp rx=0 tx=0).stop 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 shutdown_connections 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55962e572c00 0x55962e575020 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55962e4f7f30 0x55962e513a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55962e4f3180 0x55962e514e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 --2- 172.21.15.90:0/423000528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55962e4f3aa0 0x55962e513510 unknown :-1 s=CLOSED pgs=614 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 >> 172.21.15.90:0/423000528 conn(0x55962e4496a0 msgr2=0x55962e4fb150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.832+0000 7f3442ff5700 0 client.16560 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:09.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 shutdown_connections 2024-04-06T14:12:09.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f3471420b80 1 -- 172.21.15.90:0/423000528 wait complete. 2024-04-06T14:12:09.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f3471420b80 20 client.16560 trim_cache size 0 max 16384 2024-04-06T14:12:09.853 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 msgr2=0x7f34480a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 0x7f34480a3750 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f345800e420 tx=0x7f345800b040 comp rx=0 tx=0).stop 2024-04-06T14:12:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 msgr2=0x7f3460108ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.834+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 secure :-1 s=READY pgs=612 cs=0 l=1 rev1=1 crypto rx=0x7f345000b480 tx=0x7f345000b790 comp rx=0 tx=0).stop 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 shutdown_connections 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f34480a1290 0x7f34480a3750 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f34601a3680 0x7f34601a5a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3460104f70 0x7f3460108ba0 unknown :-1 s=CLOSED pgs=612 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 --2- 172.21.15.90:0/3960921588 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f34601090e0 0x7f3460109560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:09.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 >> 172.21.15.90:0/3960921588 conn(0x7f3460076d50 msgr2=0x7f3460106e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 shutdown_connections 2024-04-06T14:12:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:09.835+0000 7f346526c700 1 -- 172.21.15.90:0/3960921588 wait complete. 2024-04-06T14:12:09.856 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/683498893' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:09.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/683498893' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:10.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/683498893' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:10.858 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:12:10.858 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:12:10.858 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:12:10.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:10 smithi105 ceph-mon[46969]: pgmap v207: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 43 MiB/s wr, 2.57k op/s 2024-04-06T14:12:11.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:10 smithi090 ceph-mon[43420]: pgmap v207: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 43 MiB/s wr, 2.57k op/s 2024-04-06T14:12:11.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 -- 172.21.15.90:0/1843903191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0113f00 msgr2=0x7fb8d01162f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/1843903191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0113f00 0x7fb8d01162f0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8009f90 tx=0x7fb8c800b330 comp rx=0 tx=0).stop 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 -- 172.21.15.90:0/1843903191 shutdown_connections 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/1843903191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0113f00 0x7fb8d01162f0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/1843903191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d0107db0 0x7fb8d0108190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/1843903191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb8d01086d0 0x7fb8d01139c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.075+0000 7fb8d4f58700 1 -- 172.21.15.90:0/1843903191 >> 172.21.15.90:0/1843903191 conn(0x7fb8d006be00 msgr2=0x7fb8d006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.077+0000 7fb8d4f58700 1 -- 172.21.15.90:0/1843903191 shutdown_connections 2024-04-06T14:12:11.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.077+0000 7fb8d4f58700 1 -- 172.21.15.90:0/1843903191 wait complete. 2024-04-06T14:12:11.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 Processor -- start 2024-04-06T14:12:11.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 -- start start 2024-04-06T14:12:11.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d01086d0 0x7fb8d01a5bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb8d0113f00 0x7fb8d019f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8d019fe20 con 0x7fb8d0113f00 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8d019ff90 con 0x7fb8d0107db0 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8d4f58700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8d01a0150 con 0x7fb8d01086d0 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8ce59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8ce59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54088/0 (socket says 172.21.15.90:54088) 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.078+0000 7fb8ce59c700 1 -- 172.21.15.90:0/2621635685 learned_addr learned my addr 172.21.15.90:0/2621635685 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d01086d0 0x7fb8d01a5bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 -- 172.21.15.90:0/2621635685 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d01086d0 msgr2=0x7fb8d01a5bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d01086d0 0x7fb8d01a5bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 -- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb8d0113f00 msgr2=0x7fb8d019f7b0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb8d0113f00 0x7fb8d019f7b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 -- 172.21.15.90:0/2621635685 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb8c8009c40 con 0x7fb8d0107db0 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fb8c000c9b0 tx=0x7fb8c000cd70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8c77fe700 1 -- 172.21.15.90:0/2621635685 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c0011b60 con 0x7fb8d0107db0 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8c77fe700 1 -- 172.21.15.90:0/2621635685 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb8c0011cc0 con 0x7fb8d0107db0 2024-04-06T14:12:11.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.079+0000 7fb8c77fe700 1 -- 172.21.15.90:0/2621635685 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c0019a20 con 0x7fb8d0107db0 2024-04-06T14:12:11.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.081+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb8d01a0430 con 0x7fb8d0107db0 2024-04-06T14:12:11.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.081+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb8d0109930 con 0x7fb8d0107db0 2024-04-06T14:12:11.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.088+0000 7fb8e110cb80 1 Processor -- start 2024-04-06T14:12:11.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.092+0000 7fb8e110cb80 1 -- start start 2024-04-06T14:12:11.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56e120 0x564cdb577920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb57a250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb4c5a10 con 0x564cdb56e120 2024-04-06T14:12:11.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb4c33d0 con 0x564cdb577e60 2024-04-06T14:12:11.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.093+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb4becc0 con 0x564cdb56d800 2024-04-06T14:12:11.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.097+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.097+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:42704/0 (socket says 172.21.15.90:42704) 2024-04-06T14:12:11.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.097+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1355165283 learned_addr learned my addr 172.21.15.90:0/1355165283 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.098+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56e120 0x564cdb577920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8c77fe700 1 -- 172.21.15.90:0/2621635685 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb8c0019b80 con 0x7fb8d0107db0 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1355165283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 msgr2=0x564cdb57a250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb57a250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1355165283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56e120 msgr2=0x564cdb577920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56e120 0x564cdb577920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1355165283 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564cdb340430 con 0x564cdb56d800 2024-04-06T14:12:11.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.106+0000 7fb8c77fe700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 0x7fb8bc0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.107+0000 7fb8c77fe700 1 -- 172.21.15.90:0/2621635685 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fb8c00d2a60 con 0x7fb8d0107db0 2024-04-06T14:12:11.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.109+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 0x7fb8bc0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.110+0000 7fb8ce59c700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 0x7fb8bc0a3890 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fb8c009f8d0 tx=0x7fb8c0019440 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.117+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8009bb0 tx=0x7fb8c8007b10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e4a8e513860eeed2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.133+0000 7fb8c4ff9700 1 -- 172.21.15.90:0/1355165283 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c800e070 con 0x564cdb56d800 2024-04-06T14:12:11.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.133+0000 7fb8c4ff9700 1 -- 172.21.15.90:0/1355165283 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb8c800b9f0 con 0x564cdb56d800 2024-04-06T14:12:11.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.133+0000 7fb8c4ff9700 1 -- 172.21.15.90:0/1355165283 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c80078e0 con 0x564cdb56d800 2024-04-06T14:12:11.139 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.139+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1355165283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 msgr2=0x564cdb56dbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.139+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8009bb0 tx=0x7fb8c8007b10 comp rx=0 tx=0).stop 2024-04-06T14:12:11.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1355165283 shutdown_connections 2024-04-06T14:12:11.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb57a250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56d800 0x564cdb56dbe0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1355165283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56e120 0x564cdb577920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1355165283 >> 172.21.15.90:0/1355165283 conn(0x564cdb4cc720 msgr2=0x564cdb568b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1355165283 shutdown_connections 2024-04-06T14:12:11.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.141+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1355165283 wait complete. 2024-04-06T14:12:11.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 -- start start 2024-04-06T14:12:11.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 Processor -- start 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56d800 0x564cdb574cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb575200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb45ae00 con 0x564cdb577e60 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb45aaa0 con 0x564cdb56d800 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb342720 con 0x564cdb56e120 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.142+0000 7fb8e110cb80 10 client.0 _ll_register_callbacks cb 0x564cdb4644f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59302/0 (socket says 172.21.15.90:59302) 2024-04-06T14:12:11.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 -- 172.21.15.90:0/3720913068 learned_addr learned my addr 172.21.15.90:0/3720913068 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56d800 0x564cdb574cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 -- 172.21.15.90:0/3720913068 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 msgr2=0x564cdb575200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb575200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 -- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56d800 msgr2=0x564cdb574cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56d800 0x564cdb574cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 -- 172.21.15.90:0/3720913068 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb8d0116cc0 con 0x564cdb577e60 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7fb8c800b900 tx=0x7fb8c80073e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be902d38dcd40e23 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.143+0000 7fb8b5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:11.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c800e030 con 0x564cdb577e60 2024-04-06T14:12:11.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fb8c8024430 con 0x564cdb577e60 2024-04-06T14:12:11.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564cdb47f400 con 0x564cdb577e60 2024-04-06T14:12:11.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb8c80245f0 con 0x564cdb577e60 2024-04-06T14:12:11.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8e110cb80 10 client.16569 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:11.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.144+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564cdb2ecbe0 con 0x564cdb577e60 2024-04-06T14:12:11.147 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:10 smithi134 ceph-mon[47549]: pgmap v207: 97 pgs: 97 active+clean; 8.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 43 MiB/s wr, 2.57k op/s 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564cdb47f400 con 0x564cdb577e60 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb8c8019070 con 0x564cdb577e60 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 10 client.16569 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 10 client.16569 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 20 client.16569 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5877d0 0x564cdb589bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 4 client.16569 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564cdb4b7780 con 0x564cdb5877d0 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.145+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5877d0 0x564cdb589bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.146+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5877d0 0x564cdb589bf0 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=847b831a7fccc899 server_cookie=b41dc0e9ca544796 in_seq=0 out_seq=0 2024-04-06T14:12:11.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.146+0000 7fb8b5ffb700 10 client.16569 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.147+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/3720913068 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564cdb4b7780 con 0x564cdb5877d0 2024-04-06T14:12:11.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.147+0000 7fb8b5ffb700 10 client.16569 handle_command_reply: tid=0 2024-04-06T14:12:11.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.147+0000 7fb8e110cb80 1 client.16569 shutdown 2024-04-06T14:12:11.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 msgr2=0x564cdb575740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 secure :-1 s=READY pgs=615 cs=0 l=1 rev1=1 crypto rx=0x7fb8c800b900 tx=0x7fb8c80073e0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 shutdown_connections 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb577e60 0x564cdb575740 unknown :-1 s=CLOSED pgs=615 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56d800 0x564cdb574cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5877d0 0x564cdb589bf0 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/3720913068 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb575200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8b5ffb700 0 client.16569 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 >> 172.21.15.90:0/3720913068 conn(0x564cdb4cc720 msgr2=0x564cdb567ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 shutdown_connections 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.148+0000 7fb8e110cb80 1 -- 172.21.15.90:0/3720913068 wait complete. 2024-04-06T14:12:11.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.149+0000 7fb8e110cb80 20 client.16569 trim_cache size 0 max 16384 2024-04-06T14:12:11.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.172+0000 7fb8e110cb80 1 Processor -- start 2024-04-06T14:12:11.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.172+0000 7fb8e110cb80 1 -- start start 2024-04-06T14:12:11.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb57ea80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb56cf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb562de0 con 0x564cdb56d800 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb5762f0 con 0x564cdb577e60 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb56ea90 con 0x564cdb56e120 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59306/0 (socket says 172.21.15.90:59306) 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1108454666 learned_addr learned my addr 172.21.15.90:0/1108454666 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb56cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb57ea80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1108454666 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 msgr2=0x564cdb57ea80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb57ea80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1108454666 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 msgr2=0x564cdb56cf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb56cf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8ced9d700 1 -- 172.21.15.90:0/1108454666 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564cdb58e7e0 con 0x564cdb56d800 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.173+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb57ea80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fb8c80074b0 tx=0x7fb8c80021d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8863f46b12f9bf60 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8c57fa700 1 -- 172.21.15.90:0/1108454666 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c800e030 con 0x564cdb56d800 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8c57fa700 1 -- 172.21.15.90:0/1108454666 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb8c800f040 con 0x564cdb56d800 2024-04-06T14:12:11.176 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8c57fa700 1 -- 172.21.15.90:0/1108454666 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c8005580 con 0x564cdb56d800 2024-04-06T14:12:11.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1108454666 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 msgr2=0x564cdb573e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.174+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 secure :-1 s=READY pgs=616 cs=0 l=1 rev1=1 crypto rx=0x7fb8c80074b0 tx=0x7fb8c80021d0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.175+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1108454666 shutdown_connections 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.175+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb577e60 0x564cdb56cf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.175+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb573e20 unknown :-1 s=CLOSED pgs=616 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.175+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/1108454666 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb56e120 0x564cdb57ea80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.175+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1108454666 >> 172.21.15.90:0/1108454666 conn(0x564cdb4cc720 msgr2=0x564cdb56c7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.176+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1108454666 shutdown_connections 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 -- 172.21.15.90:0/1108454666 wait complete. 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 -- start start 2024-04-06T14:12:11.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 Processor -- start 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 0x564cdb57dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb577e60 0x564cdb5e3de0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb57e4a0 con 0x564cdb56d800 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb5e4320 con 0x564cdb56e120 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564cdb5e4460 con 0x564cdb577e60 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8e110cb80 10 client.0 _ll_register_callbacks cb 0x564cdb4644f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8ced9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59316/0 (socket says 172.21.15.90:59316) 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8ced9d700 1 -- 172.21.15.90:0/565843340 learned_addr learned my addr 172.21.15.90:0/565843340 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 0x564cdb57dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.177+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb577e60 0x564cdb5e3de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 -- 172.21.15.90:0/565843340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb577e60 msgr2=0x564cdb5e3de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb577e60 0x564cdb5e3de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 -- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 msgr2=0x564cdb57dbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 0x564cdb57dbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 -- 172.21.15.90:0/565843340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564cdb2ecbe0 con 0x564cdb56d800 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 0x564cdb57dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8ced9d700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8007480 tx=0x7fb8c8003f50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30c814f3fc8f9fde server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.178+0000 7fb8b5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb8c800e070 con 0x564cdb56d800 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fb8c8003cc0 con 0x564cdb56d800 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb8c800b9f0 con 0x564cdb56d800 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb8c8023ed0 con 0x564cdb56d800 2024-04-06T14:12:11.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 10 client.16575 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564cdb484f20 con 0x564cdb56d800 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb8c800b9f0 con 0x564cdb56d800 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb8c8019070 con 0x564cdb56d800 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 10 client.16575 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 10 client.16575 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 20 client.16575 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5e5750 0x564cdb5e7b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 4 client.16575 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:12:11.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.179+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564cdb30dd70 con 0x564cdb5e5750 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.180+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5e5750 0x564cdb5e7b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.180+0000 7fb8c7fff700 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5e5750 0x564cdb5e7b70 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=27570a3fe9686227 server_cookie=1d133833bd513a82 in_seq=0 out_seq=0 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.180+0000 7fb8b5ffb700 10 client.16575 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8b5ffb700 1 -- 172.21.15.90:0/565843340 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564cdb30dd70 con 0x564cdb5e5750 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8b5ffb700 10 client.16575 handle_command_reply: tid=0 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 client.16575 shutdown 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 msgr2=0x564cdb57d6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 secure :-1 s=READY pgs=617 cs=0 l=1 rev1=1 crypto rx=0x7fb8c8007480 tx=0x7fb8c8003f50 comp rx=0 tx=0).stop 2024-04-06T14:12:11.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 shutdown_connections 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564cdb5e5750 0x564cdb5e7b70 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564cdb577e60 0x564cdb5e3de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564cdb56d800 0x564cdb57d6a0 unknown :-1 s=CLOSED pgs=617 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 --2- 172.21.15.90:0/565843340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564cdb56e120 0x564cdb57dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 >> 172.21.15.90:0/565843340 conn(0x564cdb4cc720 msgr2=0x564cdb56c7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8b5ffb700 0 client.16575 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 shutdown_connections 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.181+0000 7fb8e110cb80 1 -- 172.21.15.90:0/565843340 wait complete. 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:11.185 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:12:11.186 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "0f3183c1-20fb-4036-9b51-4c2696106501", 2024-04-06T14:12:11.186 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:12:11.186 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.182+0000 7fb8e110cb80 20 client.16575 trim_cache size 0 max 16384 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 msgr2=0x7fb8bc0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 0x7fb8bc0a3890 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fb8c009f8d0 tx=0x7fb8c0019440 comp rx=0 tx=0).stop 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 msgr2=0x7fb8d01a5680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fb8c000c9b0 tx=0x7fb8c000cd70 comp rx=0 tx=0).stop 2024-04-06T14:12:11.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 shutdown_connections 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb8bc0a13d0 0x7fb8bc0a3890 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb8d0113f00 0x7fb8d019f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb8d0107db0 0x7fb8d01a5680 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 --2- 172.21.15.90:0/2621635685 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb8d01086d0 0x7fb8d01a5bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 >> 172.21.15.90:0/2621635685 conn(0x7fb8d006be00 msgr2=0x7fb8d0070d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 shutdown_connections 2024-04-06T14:12:11.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.186+0000 7fb8d4f58700 1 -- 172.21.15.90:0/2621635685 wait complete. 2024-04-06T14:12:11.204 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:12:11.205 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:12:11.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 -- 172.21.15.90:0/2569309437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac105da0 msgr2=0x7f0cac106180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/2569309437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac105da0 0x7f0cac106180 secure :-1 s=READY pgs=618 cs=0 l=1 rev1=1 crypto rx=0x7f0ca80099c0 tx=0x7f0ca8009cd0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 -- 172.21.15.90:0/2569309437 shutdown_connections 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/2569309437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac1008c0 0x7f0cac10cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/2569309437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac105da0 0x7f0cac106180 unknown :-1 s=CLOSED pgs=618 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/2569309437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac0fff00 0x7f0cac100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.393+0000 7f0cb46b2700 1 -- 172.21.15.90:0/2569309437 >> 172.21.15.90:0/2569309437 conn(0x7f0cac0769b0 msgr2=0x7f0cac076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.394+0000 7f0cb46b2700 1 -- 172.21.15.90:0/2569309437 shutdown_connections 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.394+0000 7f0cb46b2700 1 -- 172.21.15.90:0/2569309437 wait complete. 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.395+0000 7f0cb46b2700 1 Processor -- start 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.395+0000 7f0cb46b2700 1 -- start start 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac1008c0 0x7f0cac0728e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac105da0 0x7f0cac06b340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cac06b9f0 con 0x7f0cac0fff00 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cac06bb60 con 0x7f0cac1008c0 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb46b2700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cac06bd20 con 0x7f0cac105da0 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59342/0 (socket says 172.21.15.90:59342) 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 -- 172.21.15.90:0/4023180212 learned_addr learned my addr 172.21.15.90:0/4023180212 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb2c4f700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac105da0 0x7f0cac06b340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac1008c0 0x7f0cac0728e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 -- 172.21.15.90:0/4023180212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac105da0 msgr2=0x7f0cac06b340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac105da0 0x7f0cac06b340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 -- 172.21.15.90:0/4023180212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac1008c0 msgr2=0x7f0cac0728e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac1008c0 0x7f0cac0728e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.399 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.396+0000 7f0cb244e700 1 -- 172.21.15.90:0/4023180212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ca80095f0 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0cb244e700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f0ca800b580 tx=0x7f0ca800f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0ca37fe700 1 -- 172.21.15.90:0/4023180212 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ca801c070 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0cac06bfa0 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0ca37fe700 1 -- 172.21.15.90:0/4023180212 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0ca800fd80 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0ca37fe700 1 -- 172.21.15.90:0/4023180212 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ca8022ca0 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.397+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0cac1b2bd0 con 0x7f0cac0fff00 2024-04-06T14:12:11.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.400+0000 7f0ca37fe700 1 -- 172.21.15.90:0/4023180212 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0ca8020070 con 0x7f0cac0fff00 2024-04-06T14:12:11.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.401+0000 7f0cc0866b80 1 Processor -- start 2024-04-06T14:12:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.401+0000 7f0ca37fe700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 0x7f0c980a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.401+0000 7f0ca37fe700 1 -- 172.21.15.90:0/4023180212 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f0ca80cdb40 con 0x7f0cac0fff00 2024-04-06T14:12:11.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb2c4f700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 0x7f0c980a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 -- start start 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c542b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c54cde0 0x55f65c54f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c49a990 con 0x55f65c54cde0 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c498350 con 0x55f65c5430a0 2024-04-06T14:12:11.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c493c40 con 0x55f65c542780 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb1c4d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb2c4f700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 0x7f0c980a3750 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0ca4003e90 tx=0x7f0ca4009130 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb2c4f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c54cde0 0x55f65c54f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb1c4d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54162/0 (socket says 172.21.15.90:54162) 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.402+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/3640419491 learned_addr learned my addr 172.21.15.90:0/3640419491 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb244e700 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c542b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/3640419491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 msgr2=0x55f65c542b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c542b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/3640419491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c54cde0 msgr2=0x55f65c54f1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c54cde0 0x55f65c54f1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/3640419491 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f65c3153b0 con 0x55f65c5430a0 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c009d20 tx=0x7f0c9c011720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dc03b456fe5e1851 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0ca0ff9700 1 -- 172.21.15.90:0/3640419491 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c9c019630 con 0x55f65c5430a0 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0ca0ff9700 1 -- 172.21.15.90:0/3640419491 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c9c019c50 con 0x55f65c5430a0 2024-04-06T14:12:11.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.403+0000 7f0ca0ff9700 1 -- 172.21.15.90:0/3640419491 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c9c00ebe0 con 0x55f65c5430a0 2024-04-06T14:12:11.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3640419491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 msgr2=0x55f65c54c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c009d20 tx=0x7f0c9c011720 comp rx=0 tx=0).stop 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3640419491 shutdown_connections 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c54cde0 0x55f65c54f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c542b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3640419491 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c54c8a0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3640419491 >> 172.21.15.90:0/3640419491 conn(0x55f65c4a16a0 msgr2=0x55f65c53da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3640419491 shutdown_connections 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.404+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3640419491 wait complete. 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 -- start start 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 Processor -- start 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c549c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c54a390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c42fd80 con 0x55f65c542780 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c42fa20 con 0x55f65c5430a0 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c3176a0 con 0x55f65c54cde0 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cc0866b80 10 client.0 _ll_register_callbacks cb 0x55f65c439470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59366/0 (socket says 172.21.15.90:59366) 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cb244e700 1 -- 172.21.15.90:0/3005859880 learned_addr learned my addr 172.21.15.90:0/3005859880 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.405+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c549c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 -- 172.21.15.90:0/3005859880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 msgr2=0x55f65c54a390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c54a390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 -- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 msgr2=0x55f65c549c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c549c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 -- 172.21.15.90:0/3005859880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c9c009980 con 0x55f65c542780 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c549c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0cb244e700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f0ca8097170 tx=0x7f0ca809b4a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aa4b6f34d3b8be8b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.406+0000 7f0c95ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:11.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.407+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ca80a3060 con 0x55f65c542780 2024-04-06T14:12:11.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.407+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f0ca809b0e0 con 0x55f65c542780 2024-04-06T14:12:11.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55f65c454380 con 0x55f65c542780 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0ca809a160 con 0x55f65c542780 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 10 client.16587 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f65c2c1b60 con 0x55f65c542780 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55f65c454380 con 0x55f65c542780 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0ca80b3220 con 0x55f65c542780 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 10 client.16587 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 10 client.16587 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.408+0000 7f0cc0866b80 20 client.16587 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c56d680 0x55f65c56faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0cc0866b80 4 client.16587 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55f65c48c700 con 0x55f65c56d680 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c56d680 0x55f65c56faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0c95ffb700 10 client.16587 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.409+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c56d680 0x55f65c56faa0 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=50542fa54cb8cf56 server_cookie=28d8d652c45d277 in_seq=0 out_seq=0 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0c95ffb700 1 -- 172.21.15.90:0/3005859880 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55f65c48c700 con 0x55f65c56d680 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0c95ffb700 10 client.16587 handle_command_reply: tid=0 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 client.16587 shutdown 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 msgr2=0x55f65c54c860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 secure :-1 s=READY pgs=620 cs=0 l=1 rev1=1 crypto rx=0x7f0ca8097170 tx=0x7f0ca809b4a0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 shutdown_connections 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c56d680 0x55f65c56faa0 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c54a390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c54c860 unknown :-1 s=CLOSED pgs=620 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/3005859880 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c549c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 >> 172.21.15.90:0/3005859880 conn(0x55f65c4a16a0 msgr2=0x55f65c53cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0c95ffb700 0 client.16587 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 shutdown_connections 2024-04-06T14:12:11.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 1 -- 172.21.15.90:0/3005859880 wait complete. 2024-04-06T14:12:11.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.410+0000 7f0cc0866b80 20 client.16587 trim_cache size 0 max 16384 2024-04-06T14:12:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.434+0000 7f0cc0866b80 1 Processor -- start 2024-04-06T14:12:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 -- start start 2024-04-06T14:12:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c544f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c545480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c535330 con 0x55f65c542780 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c53e7c0 con 0x55f65c5430a0 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c569e10 con 0x55f65c54cde0 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59374/0 (socket says 172.21.15.90:59374) 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 -- 172.21.15.90:0/1645926142 learned_addr learned my addr 172.21.15.90:0/1645926142 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb2c4f700 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c545480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c544f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 -- 172.21.15.90:0/1645926142 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 msgr2=0x55f65c545480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c545480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 -- 172.21.15.90:0/1645926142 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 msgr2=0x55f65c544f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c544f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.435+0000 7f0cb244e700 1 -- 172.21.15.90:0/1645926142 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f65c459ea0 con 0x55f65c542780 2024-04-06T14:12:11.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.436+0000 7f0cb244e700 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f0ca80b3550 tx=0x7f0ca809ec20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1ef5a740a79c52b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.436+0000 7f0ca17fa700 1 -- 172.21.15.90:0/1645926142 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ca8096a60 con 0x55f65c542780 2024-04-06T14:12:11.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.436+0000 7f0ca17fa700 1 -- 172.21.15.90:0/1645926142 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0ca80a3060 con 0x55f65c542780 2024-04-06T14:12:11.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.436+0000 7f0ca17fa700 1 -- 172.21.15.90:0/1645926142 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0ca809a3c0 con 0x55f65c542780 2024-04-06T14:12:11.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 -- 172.21.15.90:0/1645926142 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 msgr2=0x55f65c549530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 secure :-1 s=READY pgs=621 cs=0 l=1 rev1=1 crypto rx=0x7f0ca80b3550 tx=0x7f0ca809ec20 comp rx=0 tx=0).stop 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 -- 172.21.15.90:0/1645926142 shutdown_connections 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c54cde0 0x55f65c545480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c542780 0x55f65c549530 unknown :-1 s=CLOSED pgs=621 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/1645926142 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c5430a0 0x55f65c544f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 -- 172.21.15.90:0/1645926142 >> 172.21.15.90:0/1645926142 conn(0x55f65c4a16a0 msgr2=0x55f65c54c3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.437+0000 7f0cc0866b80 1 -- 172.21.15.90:0/1645926142 shutdown_connections 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 -- 172.21.15.90:0/1645926142 wait complete. 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 -- start start 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 Processor -- start 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c562c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c54cde0 0x55f65c5c04b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c5c09f0 con 0x55f65c5430a0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c5c0b30 con 0x55f65c54cde0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f65c5c0ca0 con 0x55f65c542780 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cc0866b80 10 client.0 _ll_register_callbacks cb 0x55f65c439470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cb1c4d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cb244e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c562c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cb1c4d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:59382/0 (socket says 172.21.15.90:59382) 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.438+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/551180672 learned_addr learned my addr 172.21.15.90:0/551180672 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb2c4f700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c54cde0 0x55f65c5c04b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/551180672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 msgr2=0x55f65c562c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c562c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c54cde0 msgr2=0x55f65c5c04b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c54cde0 0x55f65c5c04b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 -- 172.21.15.90:0/551180672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55f65c2c1b60 con 0x55f65c5430a0 2024-04-06T14:12:11.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb244e700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c562c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cb1c4d700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c009c70 tx=0x7f0c9c00e740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3257e6d58888b127 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0c95ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c9c019dc0 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(85..85 src has 1..85) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f0c9c029040 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0c9c019c50 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0c9c009260 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cc0866b80 10 client.16593 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.439+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f65c319cd0 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c9c019c50 con 0x55f65c5430a0 2024-04-06T14:12:11.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0c9c005510 con 0x55f65c5430a0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 10 client.16593 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 10 client.16593 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 20 client.16593 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c5c23c0 0x55f65c5c47b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 4 client.16593 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55f65c2e2cf0 con 0x55f65c5c23c0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.440+0000 7f0cb244e700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c5c23c0 0x55f65c5c47b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cb244e700 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c5c23c0 0x55f65c5c47b0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=732d5650fe4824cd server_cookie=2a7960e6e3ace6a4 in_seq=0 out_seq=0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0c95ffb700 10 client.16593 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0c95ffb700 1 -- 172.21.15.90:0/551180672 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55f65c2e2cf0 con 0x55f65c5c23c0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0c95ffb700 10 client.16593 handle_command_reply: tid=0 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 client.16593 shutdown 2024-04-06T14:12:11.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 msgr2=0x55f65c563180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 secure :-1 s=READY pgs=622 cs=0 l=1 rev1=1 crypto rx=0x7f0c9c009c70 tx=0x7f0c9c00e740 comp rx=0 tx=0).stop 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 shutdown_connections 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f65c54cde0 0x55f65c5c04b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f65c5c23c0 0x55f65c5c47b0 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f65c542780 0x55f65c562c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0c95ffb700 0 client.16593 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 --2- 172.21.15.90:0/551180672 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f65c5430a0 0x55f65c563180 unknown :-1 s=CLOSED pgs=622 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 >> 172.21.15.90:0/551180672 conn(0x55f65c4a16a0 msgr2=0x55f65c541770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 shutdown_connections 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 1 -- 172.21.15.90:0/551180672 wait complete. 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.441+0000 7f0cc0866b80 20 client.16593 trim_cache size 0 max 16384 2024-04-06T14:12:11.447 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "0f3183c1-20fb-4036-9b51-4c2696106501": { 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "0f3183c1-20fb-4036-9b51-4c2696106501", 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:12:11.448 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 msgr2=0x7f0c980a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 0x7f0c980a3750 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f0ca4003e90 tx=0x7f0ca4009130 comp rx=0 tx=0).stop 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 msgr2=0x7f0cac0723a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 secure :-1 s=READY pgs=619 cs=0 l=1 rev1=1 crypto rx=0x7f0ca800b580 tx=0x7f0ca800f690 comp rx=0 tx=0).stop 2024-04-06T14:12:11.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 shutdown_connections 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c980a1290 0x7f0c980a3750 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0cac105da0 0x7f0cac06b340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0cac0fff00 0x7f0cac0723a0 unknown :-1 s=CLOSED pgs=619 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 --2- 172.21.15.90:0/4023180212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0cac1008c0 0x7f0cac0728e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 >> 172.21.15.90:0/4023180212 conn(0x7f0cac0769b0 msgr2=0x7f0cac0fed40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 shutdown_connections 2024-04-06T14:12:11.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:11.442+0000 7f0cb46b2700 1 -- 172.21.15.90:0/4023180212 wait complete. 2024-04-06T14:12:11.458 INFO:tasks.cephfs.filesystem:scrub status for tag:0f3183c1-20fb-4036-9b51-4c2696106501 - {'path': '/', 'tag': '0f3183c1-20fb-4036-9b51-4c2696106501', 'options': 'recursive,force'} 2024-04-06T14:12:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:11 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:12:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:11 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:11 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:12:11.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:11 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:11 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:12:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:11 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:12.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:11 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:12:12.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:11 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:11 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:12:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:11 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:11 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:12:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:11 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 385.0 s, 2319.8 tps, lat 43.140 ms stddev 48.841 2024-04-06T14:12:13.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:12 smithi090 ceph-mon[43420]: pgmap v208: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:13.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:12 smithi134 ceph-mon[47549]: pgmap v208: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:13.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:12 smithi105 ceph-mon[46969]: pgmap v208: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:15.150 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:14 smithi134 ceph-mon[47549]: pgmap v209: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:14 smithi105 ceph-mon[46969]: pgmap v209: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:15.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:14 smithi090 ceph-mon[43420]: pgmap v209: 97 pgs: 97 active+clean; 7.4 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 18 MiB/s wr, 1.30k op/s 2024-04-06T14:12:16.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:15 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:12:16.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:15 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:12:16.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:15 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:12:17.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:16 smithi134 ceph-mon[47549]: pgmap v210: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 662 op/s 2024-04-06T14:12:17.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:16 smithi105 ceph-mon[46969]: pgmap v210: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 662 op/s 2024-04-06T14:12:17.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:16 smithi090 ceph-mon[43420]: pgmap v210: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 662 op/s 2024-04-06T14:12:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 390.0 s, 1987.0 tps, lat 49.491 ms stddev 60.556 2024-04-06T14:12:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:18 smithi105 ceph-mon[46969]: osdmap e86: 12 total, 12 up, 12 in 2024-04-06T14:12:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:18 smithi105 ceph-mon[46969]: pgmap v212: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-04-06T14:12:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:18 smithi090 ceph-mon[43420]: osdmap e86: 12 total, 12 up, 12 in 2024-04-06T14:12:19.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:18 smithi090 ceph-mon[43420]: pgmap v212: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-04-06T14:12:19.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:18 smithi134 ceph-mon[47549]: osdmap e86: 12 total, 12 up, 12 in 2024-04-06T14:12:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:18 smithi134 ceph-mon[47549]: pgmap v212: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 661 op/s 2024-04-06T14:12:21.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:20 smithi134 ceph-mon[47549]: pgmap v213: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 655 op/s 2024-04-06T14:12:21.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:20 smithi105 ceph-mon[46969]: pgmap v213: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 655 op/s 2024-04-06T14:12:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:20 smithi090 ceph-mon[43420]: pgmap v213: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 MiB/s wr, 655 op/s 2024-04-06T14:12:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 395.0 s, 1700.8 tps, lat 58.953 ms stddev 65.799 2024-04-06T14:12:23.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:22 smithi134 ceph-mon[47549]: pgmap v214: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:23.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:22 smithi105 ceph-mon[46969]: pgmap v214: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:23.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:22 smithi090 ceph-mon[43420]: pgmap v214: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:25.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:24 smithi134 ceph-mon[47549]: pgmap v215: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:25.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:24 smithi105 ceph-mon[46969]: pgmap v215: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:25.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:24 smithi090 ceph-mon[43420]: pgmap v215: 97 pgs: 97 active+clean; 7.5 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 12 MiB/s wr, 667 op/s 2024-04-06T14:12:26.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:25 smithi134 ceph-mon[47549]: pgmap v216: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 711 op/s 2024-04-06T14:12:26.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:25 smithi105 ceph-mon[46969]: pgmap v216: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 711 op/s 2024-04-06T14:12:26.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:25 smithi090 ceph-mon[43420]: pgmap v216: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 711 op/s 2024-04-06T14:12:28.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 400.0 s, 2103.0 tps, lat 47.774 ms stddev 56.803 2024-04-06T14:12:28.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:28 smithi105 ceph-mon[46969]: pgmap v217: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 707 op/s 2024-04-06T14:12:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:28 smithi090 ceph-mon[43420]: pgmap v217: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 707 op/s 2024-04-06T14:12:29.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:28 smithi134 ceph-mon[47549]: pgmap v217: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 11 MiB/s wr, 707 op/s 2024-04-06T14:12:31.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:30 smithi134 ceph-mon[47549]: pgmap v218: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 9.2 MiB/s wr, 602 op/s 2024-04-06T14:12:31.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:30 smithi105 ceph-mon[46969]: pgmap v218: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 9.2 MiB/s wr, 602 op/s 2024-04-06T14:12:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:30 smithi090 ceph-mon[43420]: pgmap v218: 97 pgs: 97 active+clean; 7.6 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 9.2 MiB/s wr, 602 op/s 2024-04-06T14:12:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 405.0 s, 1373.0 tps, lat 73.260 ms stddev 94.855 2024-04-06T14:12:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:32 smithi134 ceph-mon[47549]: pgmap v219: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 987 op/s 2024-04-06T14:12:33.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:32 smithi105 ceph-mon[46969]: pgmap v219: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 987 op/s 2024-04-06T14:12:33.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:32 smithi090 ceph-mon[43420]: pgmap v219: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 987 op/s 2024-04-06T14:12:34.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:33 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:12:34.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:33 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:12:34.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:33 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:12:34.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:33 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:12:34.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:33 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:12:34.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:33 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:12:34.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:33 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:12:34.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:33 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:12:34.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:33 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:12:34.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:33 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:12:34.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:33 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:12:34.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:33 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:12:35.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:34 smithi134 ceph-mon[47549]: pgmap v220: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 12 MiB/s wr, 730 op/s 2024-04-06T14:12:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:34 smithi105 ceph-mon[46969]: pgmap v220: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 12 MiB/s wr, 730 op/s 2024-04-06T14:12:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:34 smithi090 ceph-mon[43420]: pgmap v220: 97 pgs: 97 active+clean; 7.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 12 MiB/s wr, 730 op/s 2024-04-06T14:12:37.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:36 smithi134 ceph-mon[47549]: pgmap v221: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 21 MiB/s wr, 1.08k op/s 2024-04-06T14:12:37.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:36 smithi105 ceph-mon[46969]: pgmap v221: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 21 MiB/s wr, 1.08k op/s 2024-04-06T14:12:37.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:36 smithi090 ceph-mon[43420]: pgmap v221: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 21 MiB/s wr, 1.08k op/s 2024-04-06T14:12:38.030 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 410.0 s, 1533.0 tps, lat 65.215 ms stddev 118.155 2024-04-06T14:12:39.007 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:38 smithi105 ceph-mon[46969]: pgmap v222: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 755 op/s 2024-04-06T14:12:39.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:38 smithi090 ceph-mon[43420]: pgmap v222: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 755 op/s 2024-04-06T14:12:39.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:38 smithi134 ceph-mon[47549]: pgmap v222: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 755 op/s 2024-04-06T14:12:41.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:40 smithi134 ceph-mon[47549]: pgmap v223: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 791 op/s 2024-04-06T14:12:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:40 smithi105 ceph-mon[46969]: pgmap v223: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 791 op/s 2024-04-06T14:12:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:40 smithi090 ceph-mon[43420]: pgmap v223: 97 pgs: 97 active+clean; 7.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 791 op/s 2024-04-06T14:12:41.459 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:12:41.460 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 -- 172.21.15.90:0/2578954516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac0ffe70 msgr2=0x7f6cac100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 --2- 172.21.15.90:0/2578954516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac0ffe70 0x7f6cac100250 secure :-1 s=READY pgs=623 cs=0 l=1 rev1=1 crypto rx=0x7f6ca4009990 tx=0x7f6ca4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 -- 172.21.15.90:0/2578954516 shutdown_connections 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 --2- 172.21.15.90:0/2578954516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac104f80 0x7f6cac10cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 --2- 172.21.15.90:0/2578954516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac0ffe70 0x7f6cac100250 unknown :-1 s=CLOSED pgs=623 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 --2- 172.21.15.90:0/2578954516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac100820 0x7f6cac104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 -- 172.21.15.90:0/2578954516 >> 172.21.15.90:0/2578954516 conn(0x7f6cac076d40 msgr2=0x7f6cac077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 -- 172.21.15.90:0/2578954516 shutdown_connections 2024-04-06T14:12:41.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.642+0000 7f6cb0819700 1 -- 172.21.15.90:0/2578954516 wait complete. 2024-04-06T14:12:41.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 Processor -- start 2024-04-06T14:12:41.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 -- start start 2024-04-06T14:12:41.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac0ffe70 0x7f6cac1980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac198b20 0x7f6cac1a3120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cac1a3800 con 0x7f6cac104f80 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cac1a3970 con 0x7f6cac198b20 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6cb0819700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6cac1a3b30 con 0x7f6cac0ffe70 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6caa59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac0ffe70 0x7f6cac1980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39694/0 (socket says 172.21.15.90:39694) 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/3028040988 learned_addr learned my addr 172.21.15.90:0/3028040988 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6caad9d700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac198b20 0x7f6cac1a3120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/3028040988 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac0ffe70 msgr2=0x7f6cac1980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac0ffe70 0x7f6cac1980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/3028040988 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac198b20 msgr2=0x7f6cac1a3120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac198b20 0x7f6cac1a3120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.643+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/3028040988 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ca40095f0 con 0x7f6cac104f80 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f6c9400e970 tx=0x7f6c9400ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6ca37fe700 1 -- 172.21.15.90:0/3028040988 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c94009790 con 0x7f6cac104f80 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6cac1a3e10 con 0x7f6cac104f80 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6cac1a42e0 con 0x7f6cac104f80 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6ca37fe700 1 -- 172.21.15.90:0/3028040988 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6c9400c7e0 con 0x7f6cac104f80 2024-04-06T14:12:41.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.644+0000 7f6ca37fe700 1 -- 172.21.15.90:0/3028040988 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c94005400 con 0x7f6cac104f80 2024-04-06T14:12:41.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.647+0000 7f6ca37fe700 1 -- 172.21.15.90:0/3028040988 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6c94021440 con 0x7f6cac104f80 2024-04-06T14:12:41.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.647+0000 7f6ca37fe700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 0x7f6c980a7be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.648+0000 7f6ca37fe700 1 -- 172.21.15.90:0/3028040988 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6c94014070 con 0x7f6cac104f80 2024-04-06T14:12:41.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.648+0000 7f6caa59c700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 0x7f6c980a7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.648+0000 7f6caa59c700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 0x7f6c980a7be0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f6ca400b2e0 tx=0x7f6ca4005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.648+0000 7f6cbc9cdb80 1 Processor -- start 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 -- start start 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d00318a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d00341d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cff7f990 con 0x55b0d0027780 2024-04-06T14:12:41.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cff7d350 con 0x55b0d00280a0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cff78c40 con 0x55b0d0031de0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caad9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d00341d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39706/0 (socket says 172.21.15.90:39706) 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 -- 172.21.15.90:0/2357127151 learned_addr learned my addr 172.21.15.90:0/2357127151 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d00318a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 -- 172.21.15.90:0/2357127151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 msgr2=0x55b0d00341d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d00341d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 -- 172.21.15.90:0/2357127151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 msgr2=0x55b0d00318a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d00318a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 -- 172.21.15.90:0/2357127151 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b0cfdfa3b0 con 0x55b0d0027780 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caad9d700 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d00341d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.649+0000 7f6caa59c700 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f6ca40095c0 tx=0x7f6ca4020000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e6d55bade7b3292 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.650+0000 7f6ca0ff9700 1 -- 172.21.15.90:0/2357127151 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ca4005660 con 0x55b0d0027780 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.650+0000 7f6ca0ff9700 1 -- 172.21.15.90:0/2357127151 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ca401e040 con 0x55b0d0027780 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.650+0000 7f6ca0ff9700 1 -- 172.21.15.90:0/2357127151 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ca400f900 con 0x55b0d0027780 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.650+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/2357127151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 msgr2=0x55b0d0027b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.650+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 secure :-1 s=READY pgs=625 cs=0 l=1 rev1=1 crypto rx=0x7f6ca40095c0 tx=0x7f6ca4020000 comp rx=0 tx=0).stop 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/2357127151 shutdown_connections 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d00341d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0027780 0x55b0d0027b60 unknown :-1 s=CLOSED pgs=625 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/2357127151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d00318a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/2357127151 >> 172.21.15.90:0/2357127151 conn(0x55b0cff866a0 msgr2=0x55b0d0022a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/2357127151 shutdown_connections 2024-04-06T14:12:41.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.651+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/2357127151 wait complete. 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 -- start start 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 Processor -- start 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 0x55b0d0031840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d002f310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cff14d80 con 0x55b0d00280a0 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cff14a20 con 0x55b0d0027780 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0cfdfc6a0 con 0x55b0d0031de0 2024-04-06T14:12:41.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6cbc9cdb80 10 client.0 _ll_register_callbacks cb 0x55b0cff1e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39712/0 (socket says 172.21.15.90:39712) 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/111531514 learned_addr learned my addr 172.21.15.90:0/111531514 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6caa59c700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 0x55b0d0031840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6caad9d700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d002f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/111531514 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 msgr2=0x55b0d002f310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d002f310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 msgr2=0x55b0d0031840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 0x55b0d0031840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6ca9d9b700 1 -- 172.21.15.90:0/111531514 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c9c011230 con 0x55b0d00280a0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.652+0000 7f6caa59c700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 0x55b0d0031840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f6c94096e70 tx=0x7f6c940850b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=db297f4996251ab0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c9409e230 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6c940a2880 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b0cff39380 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6c9409a660 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 10 client.16605 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f6cac10d370 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b0cff39380 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6c94099980 con 0x55b0d00280a0 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 10 client.16605 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 10 client.16605 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 20 client.16605 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d0041a50 0x55b0d0043e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 4 client.16605 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b0cff71700 con 0x55b0d0041a50 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.653+0000 7f6caa59c700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d0041a50 0x55b0d0043e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6c91ffb700 10 client.16605 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6caa59c700 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d0041a50 0x55b0d0043e70 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5756e57f368ff58b server_cookie=33ef00385033b267 in_seq=0 out_seq=0 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6c91ffb700 1 -- 172.21.15.90:0/111531514 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b0cff71700 con 0x55b0d0041a50 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6c91ffb700 10 client.16605 handle_command_reply: tid=0 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6cbc9cdb80 1 client.16605 shutdown 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 msgr2=0x55b0d002ebd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.654+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 secure :-1 s=READY pgs=626 cs=0 l=1 rev1=1 crypto rx=0x7f6c94096e70 tx=0x7f6c940850b0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 shutdown_connections 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d0041a50 0x55b0d0043e70 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0031de0 0x55b0d002f310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0027780 0x55b0d0031840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.662 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6c91ffb700 0 client.16605 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:41.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/111531514 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d002ebd0 unknown :-1 s=CLOSED pgs=626 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 >> 172.21.15.90:0/111531514 conn(0x55b0cff866a0 msgr2=0x55b0d0021c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 shutdown_connections 2024-04-06T14:12:41.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/111531514 wait complete. 2024-04-06T14:12:41.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.655+0000 7f6cbc9cdb80 20 client.16605 trim_cache size 0 max 16384 2024-04-06T14:12:41.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.687+0000 7f6cbc9cdb80 1 Processor -- start 2024-04-06T14:12:41.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.690+0000 7f6cbc9cdb80 1 -- start start 2024-04-06T14:12:41.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0038a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d0026e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d0029a90 con 0x55b0d00280a0 2024-04-06T14:12:41.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d00237c0 con 0x55b0d0031de0 2024-04-06T14:12:41.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.691+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d002f8f0 con 0x55b0d0027780 2024-04-06T14:12:41.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.700+0000 7f6caad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.700+0000 7f6caad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34724/0 (socket says 172.21.15.90:34724) 2024-04-06T14:12:41.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.700+0000 7f6caad9d700 1 -- 172.21.15.90:0/3801952210 learned_addr learned my addr 172.21.15.90:0/3801952210 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.701 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.700+0000 7f6caa59c700 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0038a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.704+0000 7f6caad9d700 1 -- 172.21.15.90:0/3801952210 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 msgr2=0x55b0d0038a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.704+0000 7f6caad9d700 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0038a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.704 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.704+0000 7f6caad9d700 1 -- 172.21.15.90:0/3801952210 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 msgr2=0x55b0d0026e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.704+0000 7f6caad9d700 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d0026e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.705 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.704+0000 7f6caad9d700 1 -- 172.21.15.90:0/3801952210 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b0cfda6b60 con 0x55b0d0031de0 2024-04-06T14:12:41.712 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.712+0000 7f6caad9d700 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f6c9c007630 tx=0x7f6c9c009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=eb33ec998f18b444 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.727+0000 7f6ca17fa700 1 -- 172.21.15.90:0/3801952210 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c9c00e040 con 0x55b0d0031de0 2024-04-06T14:12:41.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.727+0000 7f6ca17fa700 1 -- 172.21.15.90:0/3801952210 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6c9c00f040 con 0x55b0d0031de0 2024-04-06T14:12:41.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.727+0000 7f6ca17fa700 1 -- 172.21.15.90:0/3801952210 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c9c005260 con 0x55b0d0031de0 2024-04-06T14:12:41.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.728+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/3801952210 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 msgr2=0x55b0d002e4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.728+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f6c9c007630 tx=0x7f6c9c009920 comp rx=0 tx=0).stop 2024-04-06T14:12:41.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.733+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/3801952210 shutdown_connections 2024-04-06T14:12:41.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.733+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d0031de0 0x55b0d002e4b0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.733+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0038a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.733+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/3801952210 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d00280a0 0x55b0d0026e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.733+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/3801952210 >> 172.21.15.90:0/3801952210 conn(0x55b0cff866a0 msgr2=0x55b0d0026770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.736+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/3801952210 shutdown_connections 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.737+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/3801952210 wait complete. 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.737+0000 7f6cbc9cdb80 1 -- start start 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.737+0000 7f6cbc9cdb80 1 Processor -- start 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d0037530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0031de0 0x55b0d0037a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d0038e50 con 0x55b0d0031de0 2024-04-06T14:12:41.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d009e0a0 con 0x55b0d00280a0 2024-04-06T14:12:41.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0d00af220 con 0x55b0d0027780 2024-04-06T14:12:41.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.738+0000 7f6cbc9cdb80 10 client.0 _ll_register_callbacks cb 0x55b0cff1e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:41.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.739+0000 7f6caa59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.739+0000 7f6caa59c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49206/0 (socket says 172.21.15.90:49206) 2024-04-06T14:12:41.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.739+0000 7f6caa59c700 1 -- 172.21.15.90:0/1255735354 learned_addr learned my addr 172.21.15.90:0/1255735354 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 -- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 msgr2=0x55b0d0037530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d0037530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 -- 172.21.15.90:0/1255735354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0031de0 msgr2=0x55b0d0037a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:41.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0031de0 0x55b0d0037a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 -- 172.21.15.90:0/1255735354 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c9c009290 con 0x55b0d0027780 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6caa59c700 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f6ca4000c00 tx=0x7f6ca4005400 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ff6b6e70ce9597cc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6c91ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.740+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ca403ed40 con 0x55b0d0027780 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.741+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6ca403e1d0 con 0x55b0d0027780 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.741+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6ca401e040 con 0x55b0d0027780 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.741+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6ca403eea0 con 0x55b0d0027780 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.741+0000 7f6cbc9cdb80 10 client.26182 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:41.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.741+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b0cff3eea0 con 0x55b0d0027780 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ca401e040 con 0x55b0d0027780 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6ca40046a0 con 0x55b0d0027780 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 10 client.26182 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 10 client.26182 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 20 client.26182 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d00b05a0 0x55b0d00b29c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 4 client.26182 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:12:41.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.742+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55b0cfdc7cf0 con 0x55b0d00b05a0 2024-04-06T14:12:41.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.745+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d00b05a0 0x55b0d00b29c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.753+0000 7f6ca9d9b700 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d00b05a0 0x55b0d00b29c0 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=eec89ab3ad2a5974 server_cookie=7a1618b8990b95f4 in_seq=0 out_seq=0 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.753+0000 7f6c91ffb700 10 client.26182 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6c91ffb700 1 -- 172.21.15.90:0/1255735354 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55b0cfdc7cf0 con 0x55b0d00b05a0 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6c91ffb700 10 client.26182 handle_command_reply: tid=0 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 client.26182 shutdown 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 msgr2=0x55b0d0036ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f6ca4000c00 tx=0x7f6ca4005400 comp rx=0 tx=0).stop 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 shutdown_connections 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b0d0031de0 0x55b0d0037a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b0d00b05a0 0x55b0d00b29c0 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b0d0027780 0x55b0d0036ff0 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 --2- 172.21.15.90:0/1255735354 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b0d00280a0 0x55b0d0037530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 >> 172.21.15.90:0/1255735354 conn(0x55b0cff866a0 msgr2=0x55b0d0027000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6c91ffb700 0 client.26182 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 shutdown_connections 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.755+0000 7f6cbc9cdb80 1 -- 172.21.15.90:0/1255735354 wait complete. 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cbc9cdb80 20 client.26182 trim_cache size 0 max 16384 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:12:41.757 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:12:41.758 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:41.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 msgr2=0x7f6c980a7be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 0x7f6c980a7be0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f6ca400b2e0 tx=0x7f6ca4005fd0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 msgr2=0x7f6cac1985e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 secure :-1 s=READY pgs=624 cs=0 l=1 rev1=1 crypto rx=0x7f6c9400e970 tx=0x7f6c9400ec80 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 shutdown_connections 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6c980a5720 0x7f6c980a7be0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cac198b20 0x7f6cac1a3120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6cac0ffe70 0x7f6cac1980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 --2- 172.21.15.90:0/3028040988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6cac104f80 0x7f6cac1985e0 unknown :-1 s=CLOSED pgs=624 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 >> 172.21.15.90:0/3028040988 conn(0x7f6cac076d40 msgr2=0x7f6cac0fdfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 shutdown_connections 2024-04-06T14:12:41.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.756+0000 7f6cb0819700 1 -- 172.21.15.90:0/3028040988 wait complete. 2024-04-06T14:12:41.771 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:12:41.771 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:12:41.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 -- 172.21.15.90:0/471599801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 msgr2=0x7f4a1c109b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 --2- 172.21.15.90:0/471599801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c109b30 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f4a0c009920 tx=0x7f4a0c009c30 comp rx=0 tx=0).stop 2024-04-06T14:12:41.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 -- 172.21.15.90:0/471599801 shutdown_connections 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 --2- 172.21.15.90:0/471599801 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1150f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 --2- 172.21.15.90:0/471599801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c109b30 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 --2- 172.21.15.90:0/471599801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1127c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.946+0000 7f4a228c9700 1 -- 172.21.15.90:0/471599801 >> 172.21.15.90:0/471599801 conn(0x7f4a1c06bd30 msgr2=0x7f4a1c06c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.948+0000 7f4a228c9700 1 -- 172.21.15.90:0/471599801 shutdown_connections 2024-04-06T14:12:41.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.948+0000 7f4a228c9700 1 -- 172.21.15.90:0/471599801 wait complete. 2024-04-06T14:12:41.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.949+0000 7f4a228c9700 1 Processor -- start 2024-04-06T14:12:41.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.949+0000 7f4a228c9700 1 -- start start 2024-04-06T14:12:41.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c1a7960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1a7ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a1c1a20f0 con 0x7f4a1c10a070 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a1c1a2260 con 0x7f4a1c112d00 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a228c9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a1c1a2420 con 0x7f4a1c109750 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a20e66700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c1a7960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a20e66700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34756/0 (socket says 172.21.15.90:34756) 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a20e66700 1 -- 172.21.15.90:0/2202599541 learned_addr learned my addr 172.21.15.90:0/2202599541 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1b7fe700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1a7ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 -- 172.21.15.90:0/2202599541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 msgr2=0x7f4a1c1a1a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 -- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 msgr2=0x7f4a1c1a7ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1a7ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:41.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 -- 172.21.15.90:0/2202599541 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a14009d00 con 0x7f4a1c109750 2024-04-06T14:12:41.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1b7fe700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1a7ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:41.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a20e66700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:41.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.950+0000 7f4a1bfff700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c1a7960 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f4a0c000c00 tx=0x7f4a0c00bef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.951+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a0c01c070 con 0x7f4a1c109750 2024-04-06T14:12:41.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.951+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4a0c00bba0 con 0x7f4a1c109750 2024-04-06T14:12:41.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.951+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4a0c009520 con 0x7f4a1c109750 2024-04-06T14:12:41.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.951+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a0c0038e0 con 0x7f4a1c109750 2024-04-06T14:12:41.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.951+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4a1c1ae770 con 0x7f4a1c109750 2024-04-06T14:12:41.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.953+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4a0c020030 con 0x7f4a1c109750 2024-04-06T14:12:41.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.954+0000 7f4a197fa700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 0x7f4a040a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:41.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.954+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4a0c0cead0 con 0x7f4a1c109750 2024-04-06T14:12:41.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.954+0000 7f4a1b7fe700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 0x7f4a040a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:41.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.954+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4a080052f0 con 0x7f4a1c109750 2024-04-06T14:12:41.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.955+0000 7f4a1b7fe700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 0x7f4a040a3750 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f4a1c1a2c50 tx=0x7f4a10008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:41.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:41.960+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4a0c097f90 con 0x7f4a1c109750 2024-04-06T14:12:42.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.367+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4a080055a0 con 0x7f4a1c109750 2024-04-06T14:12:42.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.368+0000 7f4a197fa700 1 -- 172.21.15.90:0/2202599541 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f4a0c09b5b0 con 0x7f4a1c109750 2024-04-06T14:12:42.369 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:12:42.369 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:12:42.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 msgr2=0x7f4a040a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 0x7f4a040a3750 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f4a1c1a2c50 tx=0x7f4a10008040 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 msgr2=0x7f4a1c1a7960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c1a7960 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f4a0c000c00 tx=0x7f4a0c00bef0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 shutdown_connections 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4a040a1290 0x7f4a040a3750 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a1c112d00 0x7f4a1c1a1a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4a1c109750 0x7f4a1c1a7960 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 --2- 172.21.15.90:0/2202599541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4a1c10a070 0x7f4a1c1a7ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 >> 172.21.15.90:0/2202599541 conn(0x7f4a1c06bd30 msgr2=0x7f4a1c1086e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 shutdown_connections 2024-04-06T14:12:42.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.371+0000 7f4a228c9700 1 -- 172.21.15.90:0/2202599541 wait complete. 2024-04-06T14:12:42.373 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:12:42.390 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:42.391 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 -- 172.21.15.90:0/1456782556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf80ff4b0 msgr2=0x7f0bf80ff890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 --2- 172.21.15.90:0/1456782556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf80ff4b0 0x7f0bf80ff890 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0bec0098f0 tx=0x7f0bec009c00 comp rx=0 tx=0).stop 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 -- 172.21.15.90:0/1456782556 shutdown_connections 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 --2- 172.21.15.90:0/1456782556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf8106070 0x7f0bf810dc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 --2- 172.21.15.90:0/1456782556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf80ff4b0 0x7f0bf80ff890 secure :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f0bec0098f0 tx=0x7f0bec009c00 comp rx=0 tx=0).stop 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 --2- 172.21.15.90:0/1456782556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf80ffe60 0x7f0bf8105990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.614+0000 7f0bffc14700 1 -- 172.21.15.90:0/1456782556 >> 172.21.15.90:0/1456782556 conn(0x7f0bf80761b0 msgr2=0x7f0bf80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.616+0000 7f0bffc14700 1 -- 172.21.15.90:0/1456782556 shutdown_connections 2024-04-06T14:12:42.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.620+0000 7f0bffc14700 1 -- 172.21.15.90:0/1456782556 wait complete. 2024-04-06T14:12:42.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 Processor -- start 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 -- start start 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 0x7f0bf810d860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 0x7f0bf81aa250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf810b710 con 0x7f0bf810dda0 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf806a7d0 con 0x7f0bf8106070 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bffc14700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bf806a990 con 0x7f0bf810afd0 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bfd9b0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 0x7f0bf810d860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39758/0 (socket says 172.21.15.90:39758) 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bfd1af700 1 -- 172.21.15.90:0/607363119 learned_addr learned my addr 172.21.15.90:0/607363119 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.621+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 0x7f0bf81aa250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfd1af700 1 -- 172.21.15.90:0/607363119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 msgr2=0x7f0bf81aa250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 0x7f0bf81aa250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfd1af700 1 -- 172.21.15.90:0/607363119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 msgr2=0x7f0bf810d860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 0x7f0bf810d860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfd1af700 1 -- 172.21.15.90:0/607363119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bf40095f0 con 0x7f0bf810dda0 2024-04-06T14:12:42.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.622+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 0x7f0bf81aa250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:42.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.623+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 0x7f0bf810d860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:42.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.624+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f0bf400ebb0 tx=0x7f0bf400eec0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.625+0000 7f0beaffd700 1 -- 172.21.15.90:0/607363119 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf40099b0 con 0x7f0bf810dda0 2024-04-06T14:12:42.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.625+0000 7f0beaffd700 1 -- 172.21.15.90:0/607363119 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0bf40044e0 con 0x7f0bf810dda0 2024-04-06T14:12:42.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.625+0000 7f0beaffd700 1 -- 172.21.15.90:0/607363119 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf4010430 con 0x7f0bf810dda0 2024-04-06T14:12:42.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.625+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bec009520 con 0x7f0bf810dda0 2024-04-06T14:12:42.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.625+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bf806afa0 con 0x7f0bf810dda0 2024-04-06T14:12:42.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 Processor -- start 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 -- start start 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 0x55bfb17b8b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17c28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 0x55bfb17c51d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb1710990 con 0x55bfb17b8780 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb170e350 con 0x55bfb17b90a0 2024-04-06T14:12:42.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.631+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb1709c40 con 0x55bfb17c2de0 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.633+0000 7f0beaffd700 1 -- 172.21.15.90:0/607363119 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0bf4003fb0 con 0x7f0bf810dda0 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.634+0000 7f0beaffd700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 0x7f0be40a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.634+0000 7f0beaffd700 1 -- 172.21.15.90:0/607363119 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0bf4014070 con 0x7f0bf810dda0 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.634+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 0x7f0be40a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.634+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 0x55bfb17c51d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.635+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 0x7f0be40a3850 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00105b0 tx=0x7f0bf0009920 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.636+0000 7f0bfd9b0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17c28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.636+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 0x55bfb17c51d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49242/0 (socket says 172.21.15.90:49242) 2024-04-06T14:12:42.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.636+0000 7f0bfd1af700 1 -- 172.21.15.90:0/3616068222 learned_addr learned my addr 172.21.15.90:0/3616068222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 0x55bfb17b8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/3616068222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 msgr2=0x55bfb17c51d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 0x55bfb17c51d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/3616068222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 msgr2=0x55bfb17c28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17c28a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/3616068222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfb158b3b0 con 0x55bfb17b8780 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 0x55bfb17b8b60 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00093f0 tx=0x7f0bf0017000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2df53f37c83ea0a1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0be3fff700 1 -- 172.21.15.90:0/3616068222 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf000e040 con 0x55bfb17b8780 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0be3fff700 1 -- 172.21.15.90:0/3616068222 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0bf0007ba0 con 0x55bfb17b8780 2024-04-06T14:12:42.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.638+0000 7f0be3fff700 1 -- 172.21.15.90:0/3616068222 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0020680 con 0x55bfb17b8780 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/3616068222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 msgr2=0x55bfb17b8b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 0x55bfb17b8b60 secure :-1 s=READY pgs=628 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00093f0 tx=0x7f0bf0017000 comp rx=0 tx=0).stop 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/3616068222 shutdown_connections 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17c2de0 0x55bfb17c51d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b8780 0x55bfb17b8b60 unknown :-1 s=CLOSED pgs=628 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/3616068222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17c28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/3616068222 >> 172.21.15.90:0/3616068222 conn(0x55bfb17176a0 msgr2=0x55bfb17b3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.640 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/3616068222 shutdown_connections 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.639+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/3616068222 wait complete. 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 -- start start 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 Processor -- start 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b90a0 0x55bfb17c0130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17c0670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb16a5d80 con 0x55bfb17c2de0 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb16a5a20 con 0x55bfb17b8780 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb158d6a0 con 0x55bfb17b90a0 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.640+0000 7f0c0bdc8b80 10 client.0 _ll_register_callbacks cb 0x55bfb16af470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34812/0 (socket says 172.21.15.90:34812) 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 -- 172.21.15.90:0/1351735968 learned_addr learned my addr 172.21.15.90:0/1351735968 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17c0670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 -- 172.21.15.90:0/1351735968 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b90a0 msgr2=0x55bfb17c0130 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b90a0 0x55bfb17c0130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 -- 172.21.15.90:0/1351735968 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 msgr2=0x55bfb17c0670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17c0670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 -- 172.21.15.90:0/1351735968 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bf00090d0 con 0x55bfb17b8780 2024-04-06T14:12:42.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0bf80675d0 tx=0x7f0bec00b840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b66ea9cc5ce382ef server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.641+0000 7f0be17fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:42.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.642+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bec01c040 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.642+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bfb16ca380 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.642+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0bec005240 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.642+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0bec00f980 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.643+0000 7f0c0bdc8b80 10 client.26348 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.643+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bfb1537b60 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55bfb16ca380 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0bec015020 con 0x55bfb17b8780 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 10 client.26348 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 10 client.26348 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 20 client.26348 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:42.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb17d2a20 0x55bfb17d4e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 4 client.26348 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:42.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.644+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bfb1702700 con 0x55bfb17d2a20 2024-04-06T14:12:42.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.645+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb17d2a20 0x55bfb17d4e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.645+0000 7f0be17fa700 10 client.26348 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.645+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb17d2a20 0x55bfb17d4e40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=50a0626460b737da server_cookie=1812a42b7f66116b in_seq=0 out_seq=0 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0be17fa700 1 -- 172.21.15.90:0/1351735968 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55bfb1702700 con 0x55bfb17d2a20 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0be17fa700 10 client.26348 handle_command_reply: tid=0 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 client.26348 shutdown 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 msgr2=0x55bfb17bfbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0bf80675d0 tx=0x7f0bec00b840 comp rx=0 tx=0).stop 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 shutdown_connections 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb17d2a20 0x55bfb17d4e40 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17c0670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b8780 0x55bfb17bfbf0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/1351735968 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b90a0 0x55bfb17c0130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 >> 172.21.15.90:0/1351735968 conn(0x55bfb17176a0 msgr2=0x55bfb17b2c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0be17fa700 0 client.26348 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 shutdown_connections 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/1351735968 wait complete. 2024-04-06T14:12:42.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.646+0000 7f0c0bdc8b80 20 client.26348 trim_cache size 0 max 16384 2024-04-06T14:12:42.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 Processor -- start 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 -- start start 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17b4dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17baf80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17be140 con 0x55bfb17c2de0 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17afc50 con 0x55bfb17b90a0 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.663+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17afe10 con 0x55bfb17b8780 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39790/0 (socket says 172.21.15.90:39790) 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/2816632217 learned_addr learned my addr 172.21.15.90:0/2816632217 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17baf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.665 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17b4dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/2816632217 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 msgr2=0x55bfb17b4dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17b4dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/2816632217 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 msgr2=0x55bfb17baf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17baf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 -- 172.21.15.90:0/2816632217 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfb16cfea0 con 0x55bfb17c2de0 2024-04-06T14:12:42.666 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.664+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0024000 tx=0x7f0bf000ae50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bbdbe5d7d453a69 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.665+0000 7f0be8da5700 1 -- 172.21.15.90:0/2816632217 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf000e040 con 0x55bfb17c2de0 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.665+0000 7f0be8da5700 1 -- 172.21.15.90:0/2816632217 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0bf0007e50 con 0x55bfb17c2de0 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.665+0000 7f0be8da5700 1 -- 172.21.15.90:0/2816632217 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf0033e80 con 0x55bfb17c2de0 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2816632217 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 msgr2=0x55bfb17be4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 secure :-1 s=READY pgs=629 cs=0 l=1 rev1=1 crypto rx=0x7f0bf0024000 tx=0x7f0bf000ae50 comp rx=0 tx=0).stop 2024-04-06T14:12:42.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2816632217 shutdown_connections 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17c2de0 0x55bfb17be4c0 unknown :-1 s=CLOSED pgs=629 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17b4dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2816632217 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17b90a0 0x55bfb17baf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2816632217 >> 172.21.15.90:0/2816632217 conn(0x55bfb17176a0 msgr2=0x55bfb17bf480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2816632217 shutdown_connections 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2816632217 wait complete. 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.666+0000 7f0c0bdc8b80 1 -- start start 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 Processor -- start 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17c9630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 0x55bfb17c7d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17c8420 con 0x55bfb17b90a0 2024-04-06T14:12:42.669 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17c8590 con 0x55bfb17c2de0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfb17c8750 con 0x55bfb17b8780 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0c0bdc8b80 10 client.0 _ll_register_callbacks cb 0x55bfb16af470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0bfd1af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39792/0 (socket says 172.21.15.90:39792) 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0bfd1af700 1 -- 172.21.15.90:0/2738347732 learned_addr learned my addr 172.21.15.90:0/2738347732 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.667+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17c9630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 0x55bfb17c7d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfd1af700 1 -- 172.21.15.90:0/2738347732 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 msgr2=0x55bfb17c9630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17c9630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfd1af700 1 -- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 msgr2=0x55bfb17c7d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 0x55bfb17c7d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfd1af700 1 -- 172.21.15.90:0/2738347732 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55bfb158b3b0 con 0x55bfb17b90a0 2024-04-06T14:12:42.670 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.668+0000 7f0bfe1b1700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 0x55bfb17c7d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.669+0000 7f0bfd1af700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7f0bf40970d0 tx=0x7f0bf4009380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=67268feb0ccc8cfb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.669+0000 7f0be17fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf409ad60 con 0x55bfb17b90a0 2024-04-06T14:12:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0bf409a660 con 0x55bfb17b90a0 2024-04-06T14:12:42.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0bf0007ba0 con 0x55bfb17b90a0 2024-04-06T14:12:42.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0bf409e490 con 0x55bfb17b90a0 2024-04-06T14:12:42.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0c0bdc8b80 10 client.16623 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:42.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.670+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bfb158fcd0 con 0x55bfb17b90a0 2024-04-06T14:12:42.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.684+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0bf0007ba0 con 0x55bfb17b90a0 2024-04-06T14:12:42.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.684+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0bf40b5030 con 0x55bfb17b90a0 2024-04-06T14:12:42.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 10 client.16623 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:42.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 10 client.16623 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:12:42.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 20 client.16623 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:42.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb1827b70 0x55bfb1829f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 4 client.16623 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:42.688 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.687+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55bfb1558cf0 con 0x55bfb1827b70 2024-04-06T14:12:42.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.696+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb1827b70 0x55bfb1829f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.696 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.696+0000 7f0be17fa700 10 client.16623 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:42.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.696+0000 7f0bfd9b0700 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb1827b70 0x55bfb1829f90 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=47fa2c9ad3254496 server_cookie=9bbf3f90e2d1fe37 in_seq=0 out_seq=0 2024-04-06T14:12:42.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.697+0000 7f0be17fa700 1 -- 172.21.15.90:0/2738347732 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55bfb1558cf0 con 0x55bfb1827b70 2024-04-06T14:12:42.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.697+0000 7f0be17fa700 10 client.16623 handle_command_reply: tid=0 2024-04-06T14:12:42.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.697+0000 7f0c0bdc8b80 1 client.16623 shutdown 2024-04-06T14:12:42.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.699+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 msgr2=0x55bfb17c9b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.699+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 secure :-1 s=READY pgs=630 cs=0 l=1 rev1=1 crypto rx=0x7f0bf40970d0 tx=0x7f0bf4009380 comp rx=0 tx=0).stop 2024-04-06T14:12:42.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 shutdown_connections 2024-04-06T14:12:42.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55bfb1827b70 0x55bfb1829f90 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bfb17c2de0 0x55bfb17c7d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bfb17b8780 0x55bfb17c9630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 --2- 172.21.15.90:0/2738347732 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bfb17b90a0 0x55bfb17c9b70 unknown :-1 s=CLOSED pgs=630 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 >> 172.21.15.90:0/2738347732 conn(0x55bfb17176a0 msgr2=0x55bfb17be400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0be17fa700 0 client.16623 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:12:42.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 shutdown_connections 2024-04-06T14:12:42.709 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.708+0000 7f0c0bdc8b80 1 -- 172.21.15.90:0/2738347732 wait complete. 2024-04-06T14:12:42.720 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.720+0000 7f0c0bdc8b80 20 client.16623 trim_cache size 0 max 16384 2024-04-06T14:12:42.726 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:42.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.723+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 msgr2=0x7f0be40a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.723+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 0x7f0be40a3850 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f0bf00105b0 tx=0x7f0bf0009920 comp rx=0 tx=0).stop 2024-04-06T14:12:42.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.723+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 msgr2=0x7f0bf810a930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.723+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 secure :-1 s=READY pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f0bf400ebb0 tx=0x7f0bf400eec0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 shutdown_connections 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0be40a1390 0x7f0be40a3850 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0bf810afd0 0x7f0bf81aa250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0bf8106070 0x7f0bf810d860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 --2- 172.21.15.90:0/607363119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0bf810dda0 0x7f0bf810a930 secure :-1 s=CLOSED pgs=627 cs=0 l=1 rev1=1 crypto rx=0x7f0bf400ebb0 tx=0x7f0bf400eec0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.726+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 >> 172.21.15.90:0/607363119 conn(0x7f0bf80761b0 msgr2=0x7f0bf8103ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.728+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 shutdown_connections 2024-04-06T14:12:42.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.728+0000 7f0bffc14700 1 -- 172.21.15.90:0/607363119 wait complete. 2024-04-06T14:12:42.750 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:42.750 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 -- 172.21.15.90:0/854101870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b810f920 msgr2=0x7f17b81082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 --2- 172.21.15.90:0/854101870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b810f920 0x7f17b81082f0 secure :-1 s=READY pgs=631 cs=0 l=1 rev1=1 crypto rx=0x7f17b000b0a0 tx=0x7f17b000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 -- 172.21.15.90:0/854101870 shutdown_connections 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 --2- 172.21.15.90:0/854101870 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b811a590 0x7f17b811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 --2- 172.21.15.90:0/854101870 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f17b810f4b0 0x7f17b8107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 --2- 172.21.15.90:0/854101870 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b810f920 0x7f17b81082f0 unknown :-1 s=CLOSED pgs=631 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.974+0000 7f17be785700 1 -- 172.21.15.90:0/854101870 >> 172.21.15.90:0/854101870 conn(0x7f17b806be00 msgr2=0x7f17b806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.975+0000 7f17be785700 1 -- 172.21.15.90:0/854101870 shutdown_connections 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.975+0000 7f17be785700 1 -- 172.21.15.90:0/854101870 wait complete. 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.976+0000 7f17be785700 1 Processor -- start 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.976+0000 7f17be785700 1 -- start start 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f17b810f920 0x7f17b81a66f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b811a590 0x7f17b81aaa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b81ab0d0 con 0x7f17b811a590 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b81ab240 con 0x7f17b810f920 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17be785700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b81ab400 con 0x7f17b810f4b0 2024-04-06T14:12:42.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:49272/0 (socket says 172.21.15.90:49272) 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 -- 172.21.15.90:0/2341306670 learned_addr learned my addr 172.21.15.90:0/2341306670 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 -- 172.21.15.90:0/2341306670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f17b810f920 msgr2=0x7f17b81a66f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f17b810f920 0x7f17b81a66f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 -- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b811a590 msgr2=0x7f17b81aaa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b811a590 0x7f17b81aaa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.977+0000 7f17b7fff700 1 -- 172.21.15.90:0/2341306670 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17b0009d00 con 0x7f17b810f4b0 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.978+0000 7f17b7fff700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f17a800b550 tx=0x7f17a800b910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.978+0000 7f17b57fa700 1 -- 172.21.15.90:0/2341306670 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17a8011830 con 0x7f17b810f4b0 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.978+0000 7f17b57fa700 1 -- 172.21.15.90:0/2341306670 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f17a8011e50 con 0x7f17b810f4b0 2024-04-06T14:12:42.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.978+0000 7f17b57fa700 1 -- 172.21.15.90:0/2341306670 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17a800f2e0 con 0x7f17b810f4b0 2024-04-06T14:12:42.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.980+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17b81ab6e0 con 0x7f17b810f4b0 2024-04-06T14:12:42.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.980+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17b81b2f20 con 0x7f17b810f4b0 2024-04-06T14:12:42.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.982+0000 7f17b57fa700 1 -- 172.21.15.90:0/2341306670 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f17a800f440 con 0x7f17b810f4b0 2024-04-06T14:12:42.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.983+0000 7f17b57fa700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 0x7f17a00a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.983+0000 7f17b57fa700 1 -- 172.21.15.90:0/2341306670 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f17a80cd6f0 con 0x7f17b810f4b0 2024-04-06T14:12:42.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.988+0000 7f17ca939b80 1 Processor -- start 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.988+0000 7f17ca939b80 1 -- start start 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d07b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d118a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70c5f990 con 0x559c70d080a0 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70c5d350 con 0x559c70d11de0 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17ca939b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70c58c40 con 0x559c70d07780 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17b7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d07b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34850/0 (socket says 172.21.15.90:34850) 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 -- 172.21.15.90:0/589842386 learned_addr learned my addr 172.21.15.90:0/589842386 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 -- 172.21.15.90:0/589842386 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 msgr2=0x559c70d07b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d07b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 -- 172.21.15.90:0/589842386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 msgr2=0x559c70d118a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d118a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 -- 172.21.15.90:0/589842386 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559c70ada3b0 con 0x559c70d11de0 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.989+0000 7f17bcd22700 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f17ac0097a0 tx=0x7f17ac009ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=44f8f841d70294c7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.990+0000 7f179e7fc700 1 -- 172.21.15.90:0/589842386 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac011830 con 0x559c70d11de0 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.990+0000 7f179e7fc700 1 -- 172.21.15.90:0/589842386 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f17ac011e50 con 0x559c70d11de0 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.990+0000 7f179e7fc700 1 -- 172.21.15.90:0/589842386 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac00f2e0 con 0x559c70d11de0 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.991+0000 7f17ca939b80 1 -- 172.21.15.90:0/589842386 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 msgr2=0x559c70d141d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.991+0000 7f17ca939b80 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f17ac0097a0 tx=0x7f17ac009ab0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.991+0000 7f17b77fe700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 0x7f17a00a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17ca939b80 1 -- 172.21.15.90:0/589842386 shutdown_connections 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17ca939b80 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d11de0 0x559c70d141d0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17ca939b80 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d07b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17ca939b80 1 --2- 172.21.15.90:0/589842386 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d118a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17ca939b80 1 -- 172.21.15.90:0/589842386 >> 172.21.15.90:0/589842386 conn(0x559c70c666a0 msgr2=0x559c70d02a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.992+0000 7f17b77fe700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 0x7f17a00a3810 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f17b0005bd0 tx=0x7f17b0005b00 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.993+0000 7f17ca939b80 1 -- 172.21.15.90:0/589842386 shutdown_connections 2024-04-06T14:12:42.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.993+0000 7f17ca939b80 1 -- 172.21.15.90:0/589842386 wait complete. 2024-04-06T14:12:42.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.993+0000 7f17ca939b80 1 -- start start 2024-04-06T14:12:42.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.993+0000 7f17ca939b80 1 Processor -- start 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d11820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d080a0 0x559c70d0ec10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70bf4d80 con 0x559c70d11de0 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70bf4a20 con 0x559c70d07780 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70adc6a0 con 0x559c70d080a0 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17ca939b80 10 client.0 _ll_register_callbacks cb 0x559c70bfe470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17bcd22700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17bcd22700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39836/0 (socket says 172.21.15.90:39836) 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.994+0000 7f17bcd22700 1 -- 172.21.15.90:0/3450161222 learned_addr learned my addr 172.21.15.90:0/3450161222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 -- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d080a0 msgr2=0x559c70d0ec10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d080a0 0x559c70d0ec10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.997 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 -- 172.21.15.90:0/3450161222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 msgr2=0x559c70d11820 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d11820 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 -- 172.21.15.90:0/3450161222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17ac009400 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f17ac000c00 tx=0x7f17ac006d50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=689a97975843ca30 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.995+0000 7f17977fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac011630 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f17ac011790 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559c70c19380 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f17ac011950 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17ca939b80 10 client.16629 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559c70a86b60 con 0x559c70d11de0 2024-04-06T14:12:42.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.996+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559c70c19380 con 0x559c70d11de0 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f17ac04d050 con 0x559c70d11de0 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 10 client.16629 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 10 client.16629 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 20 client.16629 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d202b0 0x559c70d226d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 4 client.16629 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.997+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559c70c51700 con 0x559c70d202b0 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.998+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d202b0 0x559c70d226d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.998+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d202b0 0x559c70d226d0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c859a0c7c733b9c server_cookie=7dad554606bf4164 in_seq=0 out_seq=0 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.998+0000 7f17977fe700 10 client.16629 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.998+0000 7f17977fe700 1 -- 172.21.15.90:0/3450161222 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x559c70c51700 con 0x559c70d202b0 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.998+0000 7f17977fe700 10 client.16629 handle_command_reply: tid=0 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 client.16629 shutdown 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 msgr2=0x559c70d0f350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 secure :-1 s=READY pgs=632 cs=0 l=1 rev1=1 crypto rx=0x7f17ac000c00 tx=0x7f17ac006d50 comp rx=0 tx=0).stop 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 shutdown_connections 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d202b0 0x559c70d226d0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d0f350 unknown :-1 s=CLOSED pgs=632 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d11820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3450161222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d080a0 0x559c70d0ec10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 >> 172.21.15.90:0/3450161222 conn(0x559c70c666a0 msgr2=0x559c70d01c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17977fe700 0 client.16629 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:43.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 shutdown_connections 2024-04-06T14:12:43.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 1 -- 172.21.15.90:0/3450161222 wait complete. 2024-04-06T14:12:43.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:42.999+0000 7f17ca939b80 20 client.16629 trim_cache size 0 max 16384 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 Processor -- start 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 -- start start 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d09630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d1d240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d2eeb0 con 0x559c70d11de0 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d113b0 con 0x559c70d080a0 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.017+0000 7f17ca939b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d11b50 con 0x559c70d07780 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d1d240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17b77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17b77fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34870/0 (socket says 172.21.15.90:34870) 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17b77fe700 1 -- 172.21.15.90:0/3140611125 learned_addr learned my addr 172.21.15.90:0/3140611125 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d09630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 -- 172.21.15.90:0/3140611125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 msgr2=0x559c70d09630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d09630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 -- 172.21.15.90:0/3140611125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 msgr2=0x559c70d11610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 -- 172.21.15.90:0/3140611125 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559c70c1eea0 con 0x559c70d11de0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17b77fe700 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d1d240 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f17ac03f000 tx=0x7f17ac0096d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8c5f921afb666856 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f179effd700 1 -- 172.21.15.90:0/3140611125 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac050c50 con 0x559c70d11de0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f179effd700 1 -- 172.21.15.90:0/3140611125 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f17ac040070 con 0x559c70d11de0 2024-04-06T14:12:43.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.018+0000 7f179effd700 1 -- 172.21.15.90:0/3140611125 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17ac004310 con 0x559c70d11de0 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 -- 172.21.15.90:0/3140611125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 msgr2=0x559c70d1d240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d1d240 secure :-1 s=READY pgs=633 cs=0 l=1 rev1=1 crypto rx=0x7f17ac03f000 tx=0x7f17ac0096d0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 -- 172.21.15.90:0/3140611125 shutdown_connections 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d11de0 0x559c70d1d240 unknown :-1 s=CLOSED pgs=633 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d07780 0x559c70d09630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3140611125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d080a0 0x559c70d11610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 -- 172.21.15.90:0/3140611125 >> 172.21.15.90:0/3140611125 conn(0x559c70c666a0 msgr2=0x559c70cfe890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 -- 172.21.15.90:0/3140611125 shutdown_connections 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.019+0000 7f17ca939b80 1 -- 172.21.15.90:0/3140611125 wait complete. 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 -- start start 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 Processor -- start 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d16ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d11de0 0x559c70d17010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d17550 con 0x559c70d080a0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d176c0 con 0x559c70d07780 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559c70d17880 con 0x559c70d11de0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17ca939b80 10 client.0 _ll_register_callbacks cb 0x559c70bfe470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17b7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17b7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34874/0 (socket says 172.21.15.90:34874) 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17b7fff700 1 -- 172.21.15.90:0/3138115372 learned_addr learned my addr 172.21.15.90:0/3138115372 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17bcd22700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d11de0 0x559c70d17010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.020+0000 7f17b77fe700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d16ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 -- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d11de0 msgr2=0x559c70d17010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d11de0 0x559c70d17010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 -- 172.21.15.90:0/3138115372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 msgr2=0x559c70d16ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d16ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 -- 172.21.15.90:0/3138115372 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17b811d270 con 0x559c70d07780 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17b7fff700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f17a809a820 tx=0x7f17a800bf50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6f18422686de8cd4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17977fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.021+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17a8018510 con 0x559c70d07780 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f17a809aaf0 con 0x559c70d07780 2024-04-06T14:12:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f17ac011e50 con 0x559c70d07780 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f17a809acb0 con 0x559c70d07780 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 10 client.26363 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559c70a86b60 con 0x559c70d07780 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f17ac011e50 con 0x559c70d07780 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f17a80ba0b0 con 0x559c70d07780 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 10 client.26363 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 10 client.26363 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 20 client.26363 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d767a0 0x559c70d78b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 4 client.26363 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.022+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x559c70aa7cf0 con 0x559c70d767a0 2024-04-06T14:12:43.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17b77fe700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d767a0 0x559c70d78b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 415.0 s, 1636.8 tps, lat 60.571 ms stddev 68.267 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17977fe700 10 client.26363 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17b77fe700 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d767a0 0x559c70d78b90 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fc2d2954813804bb server_cookie=d1d188215f3de5e5 in_seq=0 out_seq=0 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17977fe700 1 -- 172.21.15.90:0/3138115372 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x559c70aa7cf0 con 0x559c70d767a0 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17977fe700 10 client.26363 handle_command_reply: tid=0 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.023+0000 7f17ca939b80 1 client.26363 shutdown 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 msgr2=0x559c70d2e830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f17a809a820 tx=0x7f17a800bf50 comp rx=0 tx=0).stop 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 shutdown_connections 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x559c70d767a0 0x559c70d78b90 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559c70d11de0 0x559c70d17010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17977fe700 0 client.26363 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:12:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559c70d07780 0x559c70d2e830 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 --2- 172.21.15.90:0/3138115372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559c70d080a0 0x559c70d16ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 >> 172.21.15.90:0/3138115372 conn(0x559c70c666a0 msgr2=0x559c70d1d240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 shutdown_connections 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 1 -- 172.21.15.90:0/3138115372 wait complete. 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.024+0000 7f17ca939b80 20 client.26363 trim_cache size 0 max 16384 2024-04-06T14:12:43.029 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 msgr2=0x7f17a00a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 0x7f17a00a3810 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f17b0005bd0 tx=0x7f17b0005b00 comp rx=0 tx=0).stop 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 msgr2=0x7f17b81a61b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f17a800b550 tx=0x7f17a800b910 comp rx=0 tx=0).stop 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 shutdown_connections 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f17a00a1350 0x7f17a00a3810 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f17b811a590 0x7f17b81aaa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f17b810f4b0 0x7f17b81a61b0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 --2- 172.21.15.90:0/2341306670 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f17b810f920 0x7f17b81a66f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 >> 172.21.15.90:0/2341306670 conn(0x7f17b806be00 msgr2=0x7f17b8118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 shutdown_connections 2024-04-06T14:12:43.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.025+0000 7f17be785700 1 -- 172.21.15.90:0/2341306670 wait complete. 2024-04-06T14:12:43.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:42 smithi090 ceph-mon[43420]: pgmap v224: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 42 MiB/s wr, 2.07k op/s 2024-04-06T14:12:43.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:42 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2202599541' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:43.044 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:12:43.044 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:12:43.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:42 smithi134 ceph-mon[47549]: pgmap v224: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 42 MiB/s wr, 2.07k op/s 2024-04-06T14:12:43.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:42 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2202599541' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 -- 172.21.15.90:0/3253561225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae41005e0 msgr2=0x7f1ae41009c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/3253561225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae41005e0 0x7f1ae41009c0 secure :-1 s=READY pgs=634 cs=0 l=1 rev1=1 crypto rx=0x7f1ae000faf0 tx=0x7f1ae000fe00 comp rx=0 tx=0).stop 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 -- 172.21.15.90:0/3253561225 shutdown_connections 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/3253561225 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae4104fd0 0x7f1ae410cb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/3253561225 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae41005e0 0x7f1ae41009c0 unknown :-1 s=CLOSED pgs=634 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/3253561225 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae4100f90 0x7f1ae41048a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.244+0000 7f1aec6fa700 1 -- 172.21.15.90:0/3253561225 >> 172.21.15.90:0/3253561225 conn(0x7f1ae40761b0 msgr2=0x7f1ae40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.245+0000 7f1aec6fa700 1 -- 172.21.15.90:0/3253561225 shutdown_connections 2024-04-06T14:12:43.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.245+0000 7f1aec6fa700 1 -- 172.21.15.90:0/3253561225 wait complete. 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.245+0000 7f1aec6fa700 1 Processor -- start 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.245+0000 7f1aec6fa700 1 -- start start 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 0x7f1ae41950d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae4104fd0 0x7f1ae4195610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae419a430 con 0x7f1ae4104fd0 2024-04-06T14:12:43.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae419a5a0 con 0x7f1ae41999b0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aec6fa700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ae419a760 con 0x7f1ae41005e0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60474/0 (socket says 172.21.15.90:60474) 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 -- 172.21.15.90:0/399821420 learned_addr learned my addr 172.21.15.90:0/399821420 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae4104fd0 0x7f1ae4195610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aea496700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 0x7f1ae41950d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 -- 172.21.15.90:0/399821420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 msgr2=0x7f1ae41950d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 0x7f1ae41950d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 -- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae4104fd0 msgr2=0x7f1ae4195610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae4104fd0 0x7f1ae4195610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 -- 172.21.15.90:0/399821420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1adc009c40 con 0x7f1ae41999b0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aea496700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 0x7f1ae41950d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.246+0000 7f1aeac97700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f1adc00c370 tx=0x7f1adc00c680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.247+0000 7f1adb7fe700 1 -- 172.21.15.90:0/399821420 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1adc017070 con 0x7f1ae41999b0 2024-04-06T14:12:43.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.247+0000 7f1adb7fe700 1 -- 172.21.15.90:0/399821420 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1adc00f040 con 0x7f1ae41999b0 2024-04-06T14:12:43.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.247+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ae000f750 con 0x7f1ae41999b0 2024-04-06T14:12:43.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.247+0000 7f1adb7fe700 1 -- 172.21.15.90:0/399821420 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1adc00e050 con 0x7f1ae41999b0 2024-04-06T14:12:43.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.247+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ae41a1fb0 con 0x7f1ae41999b0 2024-04-06T14:12:43.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:42 smithi105 ceph-mon[46969]: pgmap v224: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 42 MiB/s wr, 2.07k op/s 2024-04-06T14:12:43.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:42 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2202599541' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:12:43.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.250+0000 7f1adb7fe700 1 -- 172.21.15.90:0/399821420 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1adc004520 con 0x7f1ae41999b0 2024-04-06T14:12:43.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.253+0000 7f1af88aeb80 1 Processor -- start 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 -- start start 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1adb7fe700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 0x7f1ad00a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9b50b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b79e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec8fa990 con 0x55e4ec9b55f0 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec8f8350 con 0x55e4ec9a2780 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec8f3c40 con 0x55e4ec9a30a0 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60482/0 (socket says 172.21.15.90:60482) 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1aea496700 1 -- 172.21.15.90:0/2574284986 learned_addr learned my addr 172.21.15.90:0/2574284986 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1aeac97700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b79e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.254+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 0x7f1ad00a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9b50b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 -- 172.21.15.90:0/2574284986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 msgr2=0x55e4ec9b50b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9b50b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 -- 172.21.15.90:0/2574284986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 msgr2=0x55e4ec9b79e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b79e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 -- 172.21.15.90:0/2574284986 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e4ec7753b0 con 0x55e4ec9a2780 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9b50b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:43.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1adb7fe700 1 -- 172.21.15.90:0/399821420 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1adc0cc5d0 con 0x7f1ae41999b0 2024-04-06T14:12:43.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1aea496700 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0018000 tx=0x7f1ae000ff90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ccce97d16b6b79a1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1ad8ff9700 1 -- 172.21.15.90:0/2574284986 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0003880 con 0x55e4ec9a2780 2024-04-06T14:12:43.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1ad8ff9700 1 -- 172.21.15.90:0/2574284986 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ae00056f0 con 0x55e4ec9a2780 2024-04-06T14:12:43.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.255+0000 7f1ad8ff9700 1 -- 172.21.15.90:0/2574284986 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0003f20 con 0x55e4ec9a2780 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2574284986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 msgr2=0x55e4ec9a2b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 0x7f1ad00a3890 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4196620 tx=0x7f1ad4009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0018000 tx=0x7f1ae000ff90 comp rx=0 tx=0).stop 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2574284986 shutdown_connections 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b79e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9a2b60 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2574284986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9b50b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2574284986 >> 172.21.15.90:0/2574284986 conn(0x55e4ec9016a0 msgr2=0x55e4ec99da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2574284986 shutdown_connections 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.256+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2574284986 wait complete. 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 -- start start 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 Processor -- start 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a2780 0x55e4ec9aa190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b4a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec88fd80 con 0x55e4ec9b55f0 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec88fa20 con 0x55e4ec9a30a0 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec7776a0 con 0x55e4ec9a2780 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1af88aeb80 10 client.0 _ll_register_callbacks cb 0x55e4ec899470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1ae9c95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1aea496700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a2780 0x55e4ec9aa190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1ae9c95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60494/0 (socket says 172.21.15.90:60494) 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1ae9c95700 1 -- 172.21.15.90:0/2102664264 learned_addr learned my addr 172.21.15.90:0/2102664264 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.257+0000 7f1aeac97700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b4a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 -- 172.21.15.90:0/2102664264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a2780 msgr2=0x55e4ec9aa190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a2780 0x55e4ec9aa190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 -- 172.21.15.90:0/2102664264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 msgr2=0x55e4ec9b4a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b4a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 -- 172.21.15.90:0/2102664264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ae410d400 con 0x55e4ec9a30a0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f1ad40199e0 tx=0x7f1ad4004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bf3d7f4c3fedc2f8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1acdffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4028070 con 0x55e4ec9a30a0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1ad4004830 con 0x55e4ec9a30a0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e4ec8b4380 con 0x55e4ec9a30a0 2024-04-06T14:12:43.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1ad4010040 con 0x55e4ec9a30a0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1af88aeb80 10 client.26375 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e4ec721b60 con 0x55e4ec9a30a0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.258+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e4ec8b4380 con 0x55e4ec9a30a0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1ad402b020 con 0x55e4ec9a30a0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 10 client.26375 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 10 client.26375 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 20 client.26375 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4ec9cd9f0 0x55e4ec9cfe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 4 client.26375 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e4ec8ec700 con 0x55e4ec9cd9f0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1aea496700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4ec9cd9f0 0x55e4ec9cfe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1acdffb700 10 client.26375 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:43.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.259+0000 7f1aea496700 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4ec9cd9f0 0x55e4ec9cfe10 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e09271835a6e915b server_cookie=afa133eb422d45eb in_seq=0 out_seq=0 2024-04-06T14:12:43.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.260+0000 7f1acdffb700 1 -- 172.21.15.90:0/2102664264 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55e4ec8ec700 con 0x55e4ec9cd9f0 2024-04-06T14:12:43.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.260+0000 7f1acdffb700 10 client.26375 handle_command_reply: tid=0 2024-04-06T14:12:43.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.261+0000 7f1af88aeb80 1 client.26375 shutdown 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.261+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 msgr2=0x55e4ec9aa6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.261+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f1ad40199e0 tx=0x7f1ad4004750 comp rx=0 tx=0).stop 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 shutdown_connections 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9b4a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4ec9cd9f0 0x55e4ec9cfe10 crc :-1 s=READY pgs=139 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9a2780 0x55e4ec9aa190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/2102664264 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9aa6d0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 >> 172.21.15.90:0/2102664264 conn(0x55e4ec9016a0 msgr2=0x55e4ec99cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1acdffb700 0 client.26375 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 shutdown_connections 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.262+0000 7f1af88aeb80 1 -- 172.21.15.90:0/2102664264 wait complete. 2024-04-06T14:12:43.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.264+0000 7f1af88aeb80 20 client.26375 trim_cache size 0 max 16384 2024-04-06T14:12:43.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.286+0000 7f1af88aeb80 1 Processor -- start 2024-04-06T14:12:43.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 -- start start 2024-04-06T14:12:43.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9b4a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9a8850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9a8d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec9af3b0 con 0x55e4ec9a30a0 2024-04-06T14:12:43.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec9ab320 con 0x55e4ec9a2780 2024-04-06T14:12:43.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.292+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4ec9ae6e0 con 0x55e4ec9b55f0 2024-04-06T14:12:43.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.295+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9b4a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.295+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9b4a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60496/0 (socket says 172.21.15.90:60496) 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.295+0000 7f1aea496700 1 -- 172.21.15.90:0/812747725 learned_addr learned my addr 172.21.15.90:0/812747725 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.298+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9a8850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.299+0000 7f1ae9c95700 1 -- 172.21.15.90:0/812747725 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 msgr2=0x55e4ec9a8d90 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.299+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9a8d90 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.299+0000 7f1ae9c95700 1 -- 172.21.15.90:0/812747725 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 msgr2=0x55e4ec9b4a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.299+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9b4a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.299+0000 7f1ae9c95700 1 -- 172.21.15.90:0/812747725 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e4ec8b9ea0 con 0x55e4ec9a30a0 2024-04-06T14:12:43.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.304+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9a8850 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400b7c0 tx=0x7f1ad400bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=98542ad91a43d585 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.305+0000 7f1ad97fa700 1 -- 172.21.15.90:0/812747725 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4028070 con 0x55e4ec9a30a0 2024-04-06T14:12:43.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.305+0000 7f1ad97fa700 1 -- 172.21.15.90:0/812747725 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ad401d070 con 0x55e4ec9a30a0 2024-04-06T14:12:43.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.305+0000 7f1ad97fa700 1 -- 172.21.15.90:0/812747725 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ad4035690 con 0x55e4ec9a30a0 2024-04-06T14:12:43.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.306+0000 7f1af88aeb80 1 -- 172.21.15.90:0/812747725 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 msgr2=0x55e4ec9a8850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.306+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9a8850 secure :-1 s=READY pgs=635 cs=0 l=1 rev1=1 crypto rx=0x7f1ad400b7c0 tx=0x7f1ad400bc80 comp rx=0 tx=0).stop 2024-04-06T14:12:43.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.309+0000 7f1af88aeb80 1 -- 172.21.15.90:0/812747725 shutdown_connections 2024-04-06T14:12:43.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.309+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9a8d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.309+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9b4a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.309+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/812747725 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9a8850 unknown :-1 s=CLOSED pgs=635 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.309+0000 7f1af88aeb80 1 -- 172.21.15.90:0/812747725 >> 172.21.15.90:0/812747725 conn(0x55e4ec9016a0 msgr2=0x55e4ec9d4770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.310+0000 7f1af88aeb80 1 -- 172.21.15.90:0/812747725 shutdown_connections 2024-04-06T14:12:43.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.310+0000 7f1af88aeb80 1 -- 172.21.15.90:0/812747725 wait complete. 2024-04-06T14:12:43.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.310+0000 7f1af88aeb80 1 -- start start 2024-04-06T14:12:43.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.310+0000 7f1af88aeb80 1 Processor -- start 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9db400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9db940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4eca21040 con 0x55e4ec9a30a0 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4eca211b0 con 0x55e4ec9a2780 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.311+0000 7f1af88aeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4eca32380 con 0x55e4ec9b55f0 2024-04-06T14:12:43.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1af88aeb80 10 client.0 _ll_register_callbacks cb 0x55e4ec899470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60500/0 (socket says 172.21.15.90:60500) 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 -- 172.21.15.90:0/3938224429 learned_addr learned my addr 172.21.15.90:0/3938224429 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 -- 172.21.15.90:0/3938224429 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 msgr2=0x55e4ec9db940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9db940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 -- 172.21.15.90:0/3938224429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 msgr2=0x55e4ec9db400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:43.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9db400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.312+0000 7f1aea496700 1 -- 172.21.15.90:0/3938224429 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e4ec721b60 con 0x55e4ec9a2780 2024-04-06T14:12:43.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.313+0000 7f1aea496700 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0019040 tx=0x7f1ae0009f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=42b729648321af95 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:43.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.313+0000 7f1acdffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:43.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.314+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ae0030040 con 0x55e4ec9a2780 2024-04-06T14:12:43.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.314+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1ae0003680 con 0x55e4ec9a2780 2024-04-06T14:12:43.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.319+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1ae00056f0 con 0x55e4ec9a2780 2024-04-06T14:12:43.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.319+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1ae0003840 con 0x55e4ec9a2780 2024-04-06T14:12:43.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.321+0000 7f1af88aeb80 10 client.26381 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:43.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.321+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e4ec779cd0 con 0x55e4ec9a2780 2024-04-06T14:12:43.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ae00056f0 con 0x55e4ec9a2780 2024-04-06T14:12:43.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1ae003d070 con 0x55e4ec9a2780 2024-04-06T14:12:43.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 10 client.26381 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:43.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 10 client.26381 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:43.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 20 client.26381 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:43.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4eca337f0 0x55e4eca35c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:43.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 4 client.26381 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:12:43.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.322+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55e4ec742cf0 con 0x55e4eca337f0 2024-04-06T14:12:43.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.324+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4eca337f0 0x55e4eca35c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1acdffb700 10 client.26381 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1ae9c95700 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4eca337f0 0x55e4eca35c10 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f2ddb11025b429e1 server_cookie=63735c97c7bbf88c in_seq=0 out_seq=0 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1acdffb700 1 -- 172.21.15.90:0/3938224429 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55e4ec742cf0 con 0x55e4eca337f0 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1acdffb700 10 client.26381 handle_command_reply: tid=0 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 client.26381 shutdown 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 msgr2=0x55e4ec9daec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0019040 tx=0x7f1ae0009f00 comp rx=0 tx=0).stop 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 shutdown_connections 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e4eca337f0 0x55e4eca35c10 crc :-1 s=READY pgs=140 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e4ec9b55f0 0x55e4ec9db940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e4ec9a2780 0x55e4ec9daec0 secure :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f1ae0019040 tx=0x7f1ae0009f00 comp rx=0 tx=0).stop 2024-04-06T14:12:43.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 --2- 172.21.15.90:0/3938224429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e4ec9a30a0 0x55e4ec9db400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1acdffb700 0 client.26381 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:43.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.328+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 >> 172.21.15.90:0/3938224429 conn(0x55e4ec9016a0 msgr2=0x55e4ec9a5280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.333+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 shutdown_connections 2024-04-06T14:12:43.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.333+0000 7f1af88aeb80 1 -- 172.21.15.90:0/3938224429 wait complete. 2024-04-06T14:12:43.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.333+0000 7f1af88aeb80 20 client.26381 trim_cache size 0 max 16384 2024-04-06T14:12:43.334 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:12:43.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.334+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 msgr2=0x7f1ad00a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.334+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 0x7f1ad00a3890 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f1ae4196620 tx=0x7f1ad4009260 comp rx=0 tx=0).stop 2024-04-06T14:12:43.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.334+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 msgr2=0x7f1ae4199e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:43.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.334+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f1adc00c370 tx=0x7f1adc00c680 comp rx=0 tx=0).stop 2024-04-06T14:12:43.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 shutdown_connections 2024-04-06T14:12:43.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ad00a13d0 0x7f1ad00a3890 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.338 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1ae41999b0 0x7f1ae4199e30 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1ae41005e0 0x7f1ae41950d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 --2- 172.21.15.90:0/399821420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1ae4104fd0 0x7f1ae4195610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:43.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 >> 172.21.15.90:0/399821420 conn(0x7f1ae40761b0 msgr2=0x7f1ae41032c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:43.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 shutdown_connections 2024-04-06T14:12:43.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:43.338+0000 7f1aec6fa700 1 -- 172.21.15.90:0/399821420 wait complete. 2024-04-06T14:12:44.356 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:12:44.357 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:12:44.357 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 -- 172.21.15.90:0/498334525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40fff90 msgr2=0x7f6df4103fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/498334525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40fff90 0x7f6df4103fd0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f6de800aaa0 tx=0x7f6de800adb0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 -- 172.21.15.90:0/498334525 shutdown_connections 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/498334525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6df4104700 0x7f6df410c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/498334525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df40ff5e0 0x7f6df40ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/498334525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40fff90 0x7f6df4103fd0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 -- 172.21.15.90:0/498334525 >> 172.21.15.90:0/498334525 conn(0x7f6df40761b0 msgr2=0x7f6df40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 -- 172.21.15.90:0/498334525 shutdown_connections 2024-04-06T14:12:44.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.744+0000 7f6df8fdc700 1 -- 172.21.15.90:0/498334525 wait complete. 2024-04-06T14:12:44.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 Processor -- start 2024-04-06T14:12:44.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 -- start start 2024-04-06T14:12:44.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40ff5e0 0x7f6df4194e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6df41958f0 0x7f6df41a5430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6df41a5a00 con 0x7f6df4104700 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6df41a5b70 con 0x7f6df40ff5e0 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.745+0000 7f6df8fdc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6df41a5d30 con 0x7f6df41958f0 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34142/0 (socket says 172.21.15.90:34142) 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 -- 172.21.15.90:0/73651365 learned_addr learned my addr 172.21.15.90:0/73651365 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 -- 172.21.15.90:0/73651365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6df41958f0 msgr2=0x7f6df41a5430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6df41958f0 0x7f6df41a5430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 -- 172.21.15.90:0/73651365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40ff5e0 msgr2=0x7f6df4194e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40ff5e0 0x7f6df4194e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 -- 172.21.15.90:0/73651365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6de800a700 con 0x7f6df4104700 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f6de800ce20 tx=0x7f6de8003d70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6de37fe700 1 -- 172.21.15.90:0/73651365 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de8004350 con 0x7f6df4104700 2024-04-06T14:12:44.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.746+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6df41a5fb0 con 0x7f6df4104700 2024-04-06T14:12:44.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.747+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6df41a6430 con 0x7f6df4104700 2024-04-06T14:12:44.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.747+0000 7f6de37fe700 1 -- 172.21.15.90:0/73651365 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6de80044b0 con 0x7f6df4104700 2024-04-06T14:12:44.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.747+0000 7f6de37fe700 1 -- 172.21.15.90:0/73651365 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de8016660 con 0x7f6df4104700 2024-04-06T14:12:44.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.750+0000 7f6de37fe700 1 -- 172.21.15.90:0/73651365 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6de80167c0 con 0x7f6df4104700 2024-04-06T14:12:44.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.751+0000 7f6e05190b80 1 Processor -- start 2024-04-06T14:12:44.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 -- start start 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19891130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19893a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa197d6a10 con 0x55aa1987f120 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa197d43d0 con 0x55aa1987e800 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6e05190b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa197cfcc0 con 0x55aa19891670 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6de37fe700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 0x7f6ddc0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6df2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6df2d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60536/0 (socket says 172.21.15.90:60536) 2024-04-06T14:12:44.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6df2d9d700 1 -- 172.21.15.90:0/2151652884 learned_addr learned my addr 172.21.15.90:0/2151652884 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:44.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6de37fe700 1 -- 172.21.15.90:0/73651365 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6de8012070 con 0x7f6df4104700 2024-04-06T14:12:44.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.752+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19893a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 0x7f6ddc0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df259c700 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19891130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 -- 172.21.15.90:0/2151652884 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 msgr2=0x55aa19893a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19893a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 -- 172.21.15.90:0/2151652884 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 msgr2=0x55aa19891130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19891130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 -- 172.21.15.90:0/2151652884 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55aa19651430 con 0x55aa1987e800 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f6dec012310 tx=0x7f6dec012620 comp rx=0 tx=0).ready entity=mon.2 client_cookie=30b623a148e2a673 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.753+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 0x7f6ddc0a37d0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f6dec012a20 tx=0x7f6dec019000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.754+0000 7f6de0ff9700 1 -- 172.21.15.90:0/2151652884 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dec00bd50 con 0x55aa1987e800 2024-04-06T14:12:44.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.754+0000 7f6de0ff9700 1 -- 172.21.15.90:0/2151652884 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6dec004de0 con 0x55aa1987e800 2024-04-06T14:12:44.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.754+0000 7f6de0ff9700 1 -- 172.21.15.90:0/2151652884 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6dec021460 con 0x55aa1987e800 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.755+0000 7f6e05190b80 1 -- 172.21.15.90:0/2151652884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 msgr2=0x55aa1987ebe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.755+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f6dec012310 tx=0x7f6dec012620 comp rx=0 tx=0).stop 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.757+0000 7f6e05190b80 1 -- 172.21.15.90:0/2151652884 shutdown_connections 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.757+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19893a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.757+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa1987ebe0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.757+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2151652884 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19891130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.757+0000 7f6e05190b80 1 -- 172.21.15.90:0/2151652884 >> 172.21.15.90:0/2151652884 conn(0x55aa197dd720 msgr2=0x55aa19879b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 -- 172.21.15.90:0/2151652884 shutdown_connections 2024-04-06T14:12:44.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 -- 172.21.15.90:0/2151652884 wait complete. 2024-04-06T14:12:44.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 -- start start 2024-04-06T14:12:44.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 Processor -- start 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19891100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 0x55aa1988e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa19891670 0x55aa1988ebe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa1976be00 con 0x55aa19891670 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.760+0000 7f6e05190b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa1976baa0 con 0x55aa1987e800 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6e05190b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa19653720 con 0x55aa1987f120 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6e05190b80 10 client.0 _ll_register_callbacks cb 0x55aa197754f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19891100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19891100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60538/0 (socket says 172.21.15.90:60538) 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6df259c700 1 -- 172.21.15.90:0/2909913169 learned_addr learned my addr 172.21.15.90:0/2909913169 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:44.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.761+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 0x55aa1988e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.763+0000 7f6df1d9b700 1 -- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 msgr2=0x55aa19891100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.763+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19891100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.763+0000 7f6df1d9b700 1 -- 172.21.15.90:0/2909913169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa19891670 msgr2=0x55aa1988ebe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.763+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa19891670 0x55aa1988ebe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.763+0000 7f6df1d9b700 1 -- 172.21.15.90:0/2909913169 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6dec013040 con 0x55aa1987f120 2024-04-06T14:12:44.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.769+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 0x55aa1988e4f0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f6de80cced0 tx=0x7f6de809c9a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=12ebf29db55b50bc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.769+0000 7f6dd9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:12:44.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.772+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de80bc2d0 con 0x55aa1987f120 2024-04-06T14:12:44.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.772+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6de809cb80 con 0x55aa1987f120 2024-04-06T14:12:44.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.772+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55aa19790400 con 0x55aa1987f120 2024-04-06T14:12:44.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.775+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6de809cd40 con 0x55aa1987f120 2024-04-06T14:12:44.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.775+0000 7f6e05190b80 10 client.26227 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:44.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.775+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55aa195fdbe0 con 0x55aa1987f120 2024-04-06T14:12:44.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.775+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55aa19790400 con 0x55aa1987f120 2024-04-06T14:12:44.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6de80a4880 con 0x55aa1987f120 2024-04-06T14:12:44.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 10 client.26227 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:44.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 10 client.26227 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:44.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 20 client.26227 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:44.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198844a0 0x55aa198868c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 4 client.26227 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:44.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55aa197c8780 con 0x55aa198844a0 2024-04-06T14:12:44.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6df259c700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198844a0 0x55aa198868c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6dd9ffb700 10 client.26227 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:44.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.776+0000 7f6df259c700 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198844a0 0x55aa198868c0 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2e940738b67cb2e8 server_cookie=4c3c31355695eec3 in_seq=0 out_seq=0 2024-04-06T14:12:44.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/2909913169 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55aa197c8780 con 0x55aa198844a0 2024-04-06T14:12:44.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6dd9ffb700 10 client.26227 handle_command_reply: tid=0 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 client.26227 shutdown 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 msgr2=0x55aa1988e4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 0x55aa1988e4f0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f6de80cced0 tx=0x7f6de809c9a0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 shutdown_connections 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa19891670 0x55aa1988ebe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19891100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198844a0 0x55aa198868c0 crc :-1 s=READY pgs=141 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 --2- 172.21.15.90:0/2909913169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa1987f120 0x55aa1988e4f0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 >> 172.21.15.90:0/2909913169 conn(0x55aa197dd720 msgr2=0x55aa19878ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6dd9ffb700 0 client.26227 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.777+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 shutdown_connections 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.778+0000 7f6e05190b80 1 -- 172.21.15.90:0/2909913169 wait complete. 2024-04-06T14:12:44.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.778+0000 7f6e05190b80 20 client.26227 trim_cache size 0 max 16384 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 Processor -- start 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 -- start start 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19879960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa1988b730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa1988ca20 con 0x55aa1987f120 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa1988a060 con 0x55aa1987e800 2024-04-06T14:12:44.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6e05190b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa198b23d0 con 0x55aa19891670 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60540/0 (socket says 172.21.15.90:60540) 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.798+0000 7f6df259c700 1 -- 172.21.15.90:0/919230451 learned_addr learned my addr 172.21.15.90:0/919230451 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19879960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa1988b730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 -- 172.21.15.90:0/919230451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 msgr2=0x55aa1988b730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa1988b730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 -- 172.21.15.90:0/919230451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 msgr2=0x55aa19879960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19879960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 -- 172.21.15.90:0/919230451 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55aa198b26a0 con 0x55aa1987e800 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df2d9d700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa1988b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19879960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6df259c700 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x55aa198871a0 tx=0x7f6de40097c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d22e7dd90bd9d3cf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6de17fa700 1 -- 172.21.15.90:0/919230451 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de40184f0 con 0x55aa1987e800 2024-04-06T14:12:44.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6de17fa700 1 -- 172.21.15.90:0/919230451 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6de4012040 con 0x55aa1987e800 2024-04-06T14:12:44.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.799+0000 7f6de17fa700 1 -- 172.21.15.90:0/919230451 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de40207b0 con 0x55aa1987e800 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 -- 172.21.15.90:0/919230451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 msgr2=0x55aa19879420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x55aa198871a0 tx=0x7f6de40097c0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 -- 172.21.15.90:0/919230451 shutdown_connections 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa1988b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa19879420 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 --2- 172.21.15.90:0/919230451 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa19879960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 -- 172.21.15.90:0/919230451 >> 172.21.15.90:0/919230451 conn(0x55aa197dd720 msgr2=0x55aa1988cda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 -- 172.21.15.90:0/919230451 shutdown_connections 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.800+0000 7f6e05190b80 1 -- 172.21.15.90:0/919230451 wait complete. 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 -- start start 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 Processor -- start 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa198f83f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19897860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa19897f30 con 0x55aa1987f120 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa198980a0 con 0x55aa1987e800 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55aa19898260 con 0x55aa19891670 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6e05190b80 10 client.0 _ll_register_callbacks cb 0x55aa197754f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6df259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60548/0 (socket says 172.21.15.90:60548) 2024-04-06T14:12:44.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.801+0000 7f6df259c700 1 -- 172.21.15.90:0/700317340 learned_addr learned my addr 172.21.15.90:0/700317340 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:44.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.802+0000 7f6df259c700 1 -- 172.21.15.90:0/700317340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 msgr2=0x55aa19897860 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:12:44.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.802+0000 7f6df259c700 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19897860 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.802+0000 7f6df259c700 1 -- 172.21.15.90:0/700317340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 msgr2=0x55aa198f83f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:44.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.802+0000 7f6df259c700 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa198f83f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.802+0000 7f6df259c700 1 -- 172.21.15.90:0/700317340 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55aa19795f20 con 0x55aa1987e800 2024-04-06T14:12:44.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6df259c700 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f6de40099f0 tx=0x7f6de400beb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=66d30468b442716c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:44.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:44.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6de4018df0 con 0x55aa1987e800 2024-04-06T14:12:44.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6de402b410 con 0x55aa1987e800 2024-04-06T14:12:44.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6dec004de0 con 0x55aa1987e800 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6de402b5d0 con 0x55aa1987e800 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 10 client.26396 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55aa19655d50 con 0x55aa1987e800 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6dec004de0 con 0x55aa1987e800 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6de402a530 con 0x55aa1987e800 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 10 client.26396 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 10 client.26396 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.804+0000 7f6e05190b80 20 client.26396 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198ffc10 0x55aa19902030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6e05190b80 4 client.26396 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55aa1961ed70 con 0x55aa198ffc10 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198ffc10 0x55aa19902030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6dd9ffb700 10 client.26396 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:44.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.805+0000 7f6df1d9b700 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198ffc10 0x55aa19902030 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3e9cc331d133c5fa server_cookie=485e0bf38b4e5fb7 in_seq=0 out_seq=0 2024-04-06T14:12:44.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.813+0000 7f6dd9ffb700 1 -- 172.21.15.90:0/700317340 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55aa1961ed70 con 0x55aa198ffc10 2024-04-06T14:12:44.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.813+0000 7f6dd9ffb700 10 client.26396 handle_command_reply: tid=0 2024-04-06T14:12:44.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.818+0000 7f6e05190b80 1 client.26396 shutdown 2024-04-06T14:12:44.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.818+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 msgr2=0x55aa198f7eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.818+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f6de40099f0 tx=0x7f6de400beb0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 shutdown_connections 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55aa198ffc10 0x55aa19902030 crc :-1 s=READY pgs=142 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55aa19891670 0x55aa19897860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55aa1987e800 0x55aa198f7eb0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 --2- 172.21.15.90:0/700317340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55aa1987f120 0x55aa198f83f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 >> 172.21.15.90:0/700317340 conn(0x55aa197dd720 msgr2=0x55aa1988fc10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6dd9ffb700 0 client.26396 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:44.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 shutdown_connections 2024-04-06T14:12:44.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.822+0000 7f6e05190b80 1 -- 172.21.15.90:0/700317340 wait complete. 2024-04-06T14:12:44.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.824+0000 7f6e05190b80 20 client.26396 trim_cache size 0 max 16384 2024-04-06T14:12:44.824 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:44.825 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:12:44.825 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "905985d4-1aa8-4e22-9375-928f72725ec5", 2024-04-06T14:12:44.825 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:12:44.825 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:44.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 msgr2=0x7f6ddc0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 0x7f6ddc0a37d0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f6dec012a20 tx=0x7f6dec019000 comp rx=0 tx=0).stop 2024-04-06T14:12:44.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 msgr2=0x7f6df41953b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:44.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 secure :-1 s=READY pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f6de800ce20 tx=0x7f6de8003d70 comp rx=0 tx=0).stop 2024-04-06T14:12:44.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 shutdown_connections 2024-04-06T14:12:44.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ddc0a1310 0x7f6ddc0a37d0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6df41958f0 0x7f6df41a5430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6df40ff5e0 0x7f6df4194e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:44.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 --2- 172.21.15.90:0/73651365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6df4104700 0x7f6df41953b0 secure :-1 s=CLOSED pgs=636 cs=0 l=1 rev1=1 crypto rx=0x7f6de800ce20 tx=0x7f6de8003d70 comp rx=0 tx=0).stop 2024-04-06T14:12:44.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.828+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 >> 172.21.15.90:0/73651365 conn(0x7f6df40761b0 msgr2=0x7f6df41029a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:44.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.830+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 shutdown_connections 2024-04-06T14:12:44.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:44.831+0000 7f6df8fdc700 1 -- 172.21.15.90:0/73651365 wait complete. 2024-04-06T14:12:44.854 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:12:44.854 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 -- 172.21.15.90:0/4118739706 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff141060b0 msgr2=0x7eff1410dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 --2- 172.21.15.90:0/4118739706 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff141060b0 0x7eff1410dc30 secure :-1 s=READY pgs=637 cs=0 l=1 rev1=1 crypto rx=0x7eff0c00b0d0 tx=0x7eff0c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 -- 172.21.15.90:0/4118739706 shutdown_connections 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 --2- 172.21.15.90:0/4118739706 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff141060b0 0x7eff1410dc30 unknown :-1 s=CLOSED pgs=637 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 --2- 172.21.15.90:0/4118739706 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7eff14069590 0x7eff14069970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 --2- 172.21.15.90:0/4118739706 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7eff14069f40 0x7eff14105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 -- 172.21.15.90:0/4118739706 >> 172.21.15.90:0/4118739706 conn(0x7eff14076d50 msgr2=0x7eff14077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 -- 172.21.15.90:0/4118739706 shutdown_connections 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.021+0000 7eff1adea700 1 -- 172.21.15.90:0/4118739706 wait complete. 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 Processor -- start 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 -- start start 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7eff141060b0 0x7eff141979a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7eff1419bcf0 0x7eff1419c170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff1419c770 con 0x7eff14069590 2024-04-06T14:12:45.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff1419c8e0 con 0x7eff1419bcf0 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff1adea700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff1419caa0 con 0x7eff141060b0 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34182/0 (socket says 172.21.15.90:34182) 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.022+0000 7eff18b86700 1 -- 172.21.15.90:0/2696222835 learned_addr learned my addr 172.21.15.90:0/2696222835 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 -- 172.21.15.90:0/2696222835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7eff141060b0 msgr2=0x7eff141979a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7eff141060b0 0x7eff141979a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 -- 172.21.15.90:0/2696222835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7eff1419bcf0 msgr2=0x7eff1419c170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7eff1419bcf0 0x7eff1419c170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 -- 172.21.15.90:0/2696222835 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff0c009d00 con 0x7eff14069590 2024-04-06T14:12:45.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff18b86700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7eff0800b530 tx=0x7eff0800b8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff11ffb700 1 -- 172.21.15.90:0/2696222835 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff08011830 con 0x7eff14069590 2024-04-06T14:12:45.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff11ffb700 1 -- 172.21.15.90:0/2696222835 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7eff08011e50 con 0x7eff14069590 2024-04-06T14:12:45.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff141a3fa0 con 0x7eff14069590 2024-04-06T14:12:45.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff11ffb700 1 -- 172.21.15.90:0/2696222835 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff0800f2e0 con 0x7eff14069590 2024-04-06T14:12:45.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.023+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff141a4480 con 0x7eff14069590 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.025+0000 7eff11ffb700 1 -- 172.21.15.90:0/2696222835 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7eff08010580 con 0x7eff14069590 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.026+0000 7eff11ffb700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 0x7eff040a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.026+0000 7eff11ffb700 1 -- 172.21.15.90:0/2696222835 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7eff080cc720 con 0x7eff14069590 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.026+0000 7eff13fff700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 0x7eff040a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.026+0000 7eff13fff700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 0x7eff040a3750 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7eff14198960 tx=0x7eff00006d20 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 Processor -- start 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 -- start start 2024-04-06T14:12:45.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834c78a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834ca1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc83415990 con 0x55dc834bd780 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc83413350 con 0x55dc834c7de0 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc8340ec40 con 0x55dc834be0a0 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34192/0 (socket says 172.21.15.90:34192) 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 -- 172.21.15.90:0/2324155844 learned_addr learned my addr 172.21.15.90:0/2324155844 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 -- 172.21.15.90:0/2324155844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 msgr2=0x55dc834c78a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834c78a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 -- 172.21.15.90:0/2324155844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 msgr2=0x55dc834ca1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834ca1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 -- 172.21.15.90:0/2324155844 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dc832903b0 con 0x55dc834bd780 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.027+0000 7eff18b86700 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7eff08095e20 tx=0x7eff08099380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d4471a2ebef708d2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7efefeffd700 1 -- 172.21.15.90:0/2324155844 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff0809d750 con 0x55dc834bd780 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7efefeffd700 1 -- 172.21.15.90:0/2324155844 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7eff080a1da0 con 0x55dc834bd780 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7efefeffd700 1 -- 172.21.15.90:0/2324155844 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff080990e0 con 0x55dc834bd780 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 -- 172.21.15.90:0/2324155844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 msgr2=0x55dc834bdb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 secure :-1 s=READY pgs=639 cs=0 l=1 rev1=1 crypto rx=0x7eff08095e20 tx=0x7eff08099380 comp rx=0 tx=0).stop 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 -- 172.21.15.90:0/2324155844 shutdown_connections 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834ca1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834bdb60 unknown :-1 s=CLOSED pgs=639 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/2324155844 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834c78a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 -- 172.21.15.90:0/2324155844 >> 172.21.15.90:0/2324155844 conn(0x55dc8341c6a0 msgr2=0x55dc834b8a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 -- 172.21.15.90:0/2324155844 shutdown_connections 2024-04-06T14:12:45.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.028+0000 7eff26f9eb80 1 -- 172.21.15.90:0/2324155844 wait complete. 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 -- start start 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 Processor -- start 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834be0a0 0x55dc834c4ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834c7de0 0x55dc834c5290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc833aad80 con 0x55dc834bd780 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc833aaa20 con 0x55dc834be0a0 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc832926a0 con 0x55dc834c7de0 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff26f9eb80 10 client.0 _ll_register_callbacks cb 0x55dc833b4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34202/0 (socket says 172.21.15.90:34202) 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 -- 172.21.15.90:0/6462972 learned_addr learned my addr 172.21.15.90:0/6462972 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:45.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff13fff700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834be0a0 0x55dc834c4ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 -- 172.21.15.90:0/6462972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834c7de0 msgr2=0x55dc834c5290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834c7de0 0x55dc834c5290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 -- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834be0a0 msgr2=0x55dc834c4ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834be0a0 0x55dc834c4ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.029+0000 7eff18b86700 1 -- 172.21.15.90:0/6462972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff141a4480 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff18b86700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7eff080ca3d0 tx=0x7eff08099e60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=571770f82b973144 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff0809d750 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7eff08098e10 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55dc833cf380 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7eff08065320 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 10 client.16671 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dc8323cb60 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55dc833cf380 con 0x55dc834bd780 2024-04-06T14:12:45.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7eff08095710 con 0x55dc834bd780 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 10 client.16671 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 10 client.16671 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 20 client.16671 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc834d7720 0x55dc834d9b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 4 client.16671 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:12:45.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.030+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55dc83407700 con 0x55dc834d7720 2024-04-06T14:12:45.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.050+0000 7eff13fff700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc834d7720 0x55dc834d9b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.051+0000 7efef3fff700 10 client.16671 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:45.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.051+0000 7eff13fff700 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc834d7720 0x55dc834d9b10 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=946f826ef7fce4e3 server_cookie=5aedf63d8dcc0a25 in_seq=0 out_seq=0 2024-04-06T14:12:45.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7efef3fff700 1 -- 172.21.15.90:0/6462972 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55dc83407700 con 0x55dc834d7720 2024-04-06T14:12:45.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7efef3fff700 10 client.16671 handle_command_reply: tid=0 2024-04-06T14:12:45.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 client.16671 shutdown 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 msgr2=0x55dc834c7860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 secure :-1 s=READY pgs=640 cs=0 l=1 rev1=1 crypto rx=0x7eff080ca3d0 tx=0x7eff08099e60 comp rx=0 tx=0).stop 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 shutdown_connections 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc834d7720 0x55dc834d9b10 crc :-1 s=READY pgs=143 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834c7de0 0x55dc834c5290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834c7860 unknown :-1 s=CLOSED pgs=640 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7efef3fff700 0 client.16671 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/6462972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834be0a0 0x55dc834c4ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.057+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 >> 172.21.15.90:0/6462972 conn(0x55dc8341c6a0 msgr2=0x55dc834b7c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.058+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 shutdown_connections 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.058+0000 7eff26f9eb80 1 -- 172.21.15.90:0/6462972 wait complete. 2024-04-06T14:12:45.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.058+0000 7eff26f9eb80 20 client.16671 trim_cache size 0 max 16384 2024-04-06T14:12:45.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 Processor -- start 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 -- start start 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834b8520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834b8a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc834b8fa0 con 0x55dc834bd780 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc834c3170 con 0x55dc834c7de0 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc834c07d0 con 0x55dc834be0a0 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.074+0000 7eff18b86700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34206/0 (socket says 172.21.15.90:34206) 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 -- 172.21.15.90:0/1743623837 learned_addr learned my addr 172.21.15.90:0/1743623837 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff13fff700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834b8520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff19387700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834b8a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 -- 172.21.15.90:0/1743623837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 msgr2=0x55dc834b8520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834b8520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 -- 172.21.15.90:0/1743623837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 msgr2=0x55dc834b8a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834b8a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 -- 172.21.15.90:0/1743623837 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dc833d4ea0 con 0x55dc834bd780 2024-04-06T14:12:45.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff19387700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834b8a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:12:45.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.075+0000 7eff18b86700 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7eff08096390 tx=0x7eff08099590 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5f765f5d2b05d20c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.083 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.083+0000 7efeff7fe700 1 -- 172.21.15.90:0/1743623837 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff0800d040 con 0x55dc834bd780 2024-04-06T14:12:45.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.083+0000 7efeff7fe700 1 -- 172.21.15.90:0/1743623837 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7eff08065320 con 0x55dc834bd780 2024-04-06T14:12:45.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.083+0000 7efeff7fe700 1 -- 172.21.15.90:0/1743623837 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff08078a00 con 0x55dc834bd780 2024-04-06T14:12:45.084 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.084+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1743623837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 msgr2=0x55dc834cea60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.085 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.084+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 secure :-1 s=READY pgs=641 cs=0 l=1 rev1=1 crypto rx=0x7eff08096390 tx=0x7eff08099590 comp rx=0 tx=0).stop 2024-04-06T14:12:45.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.088+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1743623837 shutdown_connections 2024-04-06T14:12:45.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.088+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834b8a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.088+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834bd780 0x55dc834cea60 unknown :-1 s=CLOSED pgs=641 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.088+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1743623837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834be0a0 0x55dc834b8520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.088+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1743623837 >> 172.21.15.90:0/1743623837 conn(0x55dc8341c6a0 msgr2=0x55dc834c34f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.097+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1743623837 shutdown_connections 2024-04-06T14:12:45.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.099+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1743623837 wait complete. 2024-04-06T14:12:45.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.100+0000 7eff26f9eb80 1 -- start start 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.100+0000 7eff26f9eb80 1 Processor -- start 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834bd780 0x55dc834ce9d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834be0a0 0x55dc834cce70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc834cdc90 con 0x55dc834be0a0 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc83540950 con 0x55dc834c7de0 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dc83540a90 con 0x55dc834bd780 2024-04-06T14:12:45.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.101+0000 7eff26f9eb80 10 client.0 _ll_register_callbacks cb 0x7eff080994e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:60618/0 (socket says 172.21.15.90:60618) 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 -- 172.21.15.90:0/1785990331 learned_addr learned my addr 172.21.15.90:0/1785990331 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 -- 172.21.15.90:0/1785990331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834bd780 msgr2=0x55dc834ce9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834bd780 0x55dc834ce9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 -- 172.21.15.90:0/1785990331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834be0a0 msgr2=0x55dc834cce70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:12:45.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834be0a0 0x55dc834cce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 -- 172.21.15.90:0/1785990331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55dc8323cb60 con 0x55dc834c7de0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7eff19387700 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7eff0c015000 tx=0x7eff0c009180 comp rx=0 tx=0).ready entity=mon.2 client_cookie=da3556ff5ef354ac server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.102+0000 7efef3fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.103+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7eff0c00e050 con 0x55dc834c7de0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.103+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(86..86 src has 1..86) v4 ==== 8556+0+0 (secure 0 0 0) 0x7eff0c027070 con 0x55dc834c7de0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.103+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55dc8325dcf0 con 0x55dc834c7de0 2024-04-06T14:12:45.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.103+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7eff0c021d30 con 0x55dc834c7de0 2024-04-06T14:12:45.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.111+0000 7eff26f9eb80 10 client.26402 fetch_fsmap learned FSMap version 24 2024-04-06T14:12:45.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.111+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dc83294cd0 con 0x55dc834c7de0 2024-04-06T14:12:45.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.117+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55dc8325dcf0 con 0x55dc834c7de0 2024-04-06T14:12:45.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.118+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7eff0c00b100 con 0x55dc834c7de0 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.121+0000 7eff26f9eb80 10 client.26402 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.121+0000 7eff26f9eb80 10 client.26402 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.121+0000 7eff26f9eb80 20 client.26402 populate_metadata read hostname 'smithi090' 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.121+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc835219a0 0x55dc83523dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.122+0000 7eff26f9eb80 4 client.26402 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:12:45.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.122+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55dc83292ce0 con 0x55dc835219a0 2024-04-06T14:12:45.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.126+0000 7eff18b86700 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc835219a0 0x55dc83523dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:12:45.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.134+0000 7eff18b86700 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc835219a0 0x55dc83523dc0 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f8d14599831f7c3b server_cookie=9141d97f2f573c40 in_seq=0 out_seq=0 2024-04-06T14:12:45.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.134+0000 7efef3fff700 10 client.26402 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:45.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.134+0000 7efef3fff700 1 -- 172.21.15.90:0/1785990331 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x55dc83292ce0 con 0x55dc835219a0 2024-04-06T14:12:45.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.134+0000 7efef3fff700 10 client.26402 handle_command_reply: tid=0 2024-04-06T14:12:45.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.137+0000 7eff26f9eb80 1 client.26402 shutdown 2024-04-06T14:12:45.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:44 smithi134 ceph-mon[47549]: pgmap v225: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 34 MiB/s wr, 1.67k op/s 2024-04-06T14:12:45.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.139+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 msgr2=0x55dc834cd5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.139+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7eff0c015000 tx=0x7eff0c009180 comp rx=0 tx=0).stop 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 shutdown_connections 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dc834c7de0 0x55dc834cd5b0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dc835219a0 0x55dc83523dc0 crc :-1 s=READY pgs=144 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dc834bd780 0x55dc834ce9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 --2- 172.21.15.90:0/1785990331 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dc834be0a0 0x55dc834cce70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.144+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 >> 172.21.15.90:0/1785990331 conn(0x55dc8341c6a0 msgr2=0x55dc834c34b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.145+0000 7efef3fff700 0 client.26402 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:12:45.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.146+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 shutdown_connections 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (626 inodes in the stack)", 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "905985d4-1aa8-4e22-9375-928f72725ec5": { 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "905985d4-1aa8-4e22-9375-928f72725ec5", 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:12:45.153 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:12:45.154 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:12:45.154 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:12:45.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.148+0000 7eff26f9eb80 1 -- 172.21.15.90:0/1785990331 wait complete. 2024-04-06T14:12:45.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.149+0000 7eff26f9eb80 20 client.26402 trim_cache size 0 max 16384 2024-04-06T14:12:45.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.158+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 msgr2=0x7eff040a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.158+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 0x7eff040a3750 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7eff14198960 tx=0x7eff00006d20 comp rx=0 tx=0).stop 2024-04-06T14:12:45.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.158+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 msgr2=0x7eff14197460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:12:45.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.158+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 secure :-1 s=READY pgs=638 cs=0 l=1 rev1=1 crypto rx=0x7eff0800b530 tx=0x7eff0800b8f0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 shutdown_connections 2024-04-06T14:12:45.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7eff040a1290 0x7eff040a3750 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7eff1419bcf0 0x7eff1419c170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7eff14069590 0x7eff14197460 unknown :-1 s=CLOSED pgs=638 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 --2- 172.21.15.90:0/2696222835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7eff141060b0 0x7eff141979a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:12:45.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 >> 172.21.15.90:0/2696222835 conn(0x7eff14076d50 msgr2=0x7eff140ff380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:12:45.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 shutdown_connections 2024-04-06T14:12:45.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:12:45.160+0000 7eff1adea700 1 -- 172.21.15.90:0/2696222835 wait complete. 2024-04-06T14:12:45.181 INFO:tasks.cephfs.filesystem:scrub status for tag:905985d4-1aa8-4e22-9375-928f72725ec5 - {'path': '/', 'tag': '905985d4-1aa8-4e22-9375-928f72725ec5', 'options': 'recursive,force'} 2024-04-06T14:12:45.181 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:44 smithi090 ceph-mon[43420]: pgmap v225: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 34 MiB/s wr, 1.67k op/s 2024-04-06T14:12:45.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:44 smithi105 ceph-mon[46969]: pgmap v225: 97 pgs: 97 active+clean; 8.0 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 34 MiB/s wr, 1.67k op/s 2024-04-06T14:12:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:45 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:12:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:45 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:45 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:12:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:45 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:45 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:12:46.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:45 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:12:46.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:45 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:45 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:12:46.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:45 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:45 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:12:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:45 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:12:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:45 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:45 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:12:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:45 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:12:46.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:45 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:12:47.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:46 smithi105 ceph-mon[46969]: pgmap v226: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 48 MiB/s wr, 2.72k op/s 2024-04-06T14:12:47.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:46 smithi090 ceph-mon[43420]: pgmap v226: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 48 MiB/s wr, 2.72k op/s 2024-04-06T14:12:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:46 smithi134 ceph-mon[47549]: pgmap v226: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 48 MiB/s wr, 2.72k op/s 2024-04-06T14:12:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 420.0 s, 1635.8 tps, lat 61.709 ms stddev 97.880 2024-04-06T14:12:48.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:48 smithi105 ceph-mon[46969]: pgmap v227: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 40 MiB/s wr, 2.37k op/s 2024-04-06T14:12:49.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:48 smithi090 ceph-mon[43420]: pgmap v227: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 40 MiB/s wr, 2.37k op/s 2024-04-06T14:12:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:48 smithi134 ceph-mon[47549]: pgmap v227: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 40 MiB/s wr, 2.37k op/s 2024-04-06T14:12:51.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:50 smithi134 ceph-mon[47549]: pgmap v228: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 41 MiB/s wr, 2.39k op/s 2024-04-06T14:12:51.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:50 smithi105 ceph-mon[46969]: pgmap v228: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 41 MiB/s wr, 2.39k op/s 2024-04-06T14:12:51.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:50 smithi090 ceph-mon[43420]: pgmap v228: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 41 MiB/s wr, 2.39k op/s 2024-04-06T14:12:52.255 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:51 smithi105 ceph-mon[46969]: pgmap v229: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 48 MiB/s wr, 2.70k op/s 2024-04-06T14:12:52.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:51 smithi090 ceph-mon[43420]: pgmap v229: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 48 MiB/s wr, 2.70k op/s 2024-04-06T14:12:52.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:51 smithi134 ceph-mon[47549]: pgmap v229: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 48 MiB/s wr, 2.70k op/s 2024-04-06T14:12:53.030 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 425.0 s, 2102.2 tps, lat 47.696 ms stddev 54.180 2024-04-06T14:12:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:54 smithi134 ceph-mon[47549]: pgmap v230: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 179 KiB/s rd, 22 MiB/s wr, 1.41k op/s 2024-04-06T14:12:55.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:54 smithi105 ceph-mon[46969]: pgmap v230: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 179 KiB/s rd, 22 MiB/s wr, 1.41k op/s 2024-04-06T14:12:55.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:54 smithi090 ceph-mon[43420]: pgmap v230: 97 pgs: 97 active+clean; 8.1 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 179 KiB/s rd, 22 MiB/s wr, 1.41k op/s 2024-04-06T14:12:56.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:55 smithi105 ceph-mon[46969]: pgmap v231: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 181 KiB/s rd, 29 MiB/s wr, 1.78k op/s 2024-04-06T14:12:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:55 smithi090 ceph-mon[43420]: pgmap v231: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 181 KiB/s rd, 29 MiB/s wr, 1.78k op/s 2024-04-06T14:12:56.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:55 smithi134 ceph-mon[47549]: pgmap v231: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 181 KiB/s rd, 29 MiB/s wr, 1.78k op/s 2024-04-06T14:12:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 430.0 s, 2102.9 tps, lat 47.465 ms stddev 53.357 2024-04-06T14:12:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:12:58 smithi090 ceph-mon[43420]: pgmap v232: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 730 op/s 2024-04-06T14:12:59.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:12:58 smithi134 ceph-mon[47549]: pgmap v232: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 730 op/s 2024-04-06T14:12:59.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:12:58 smithi105 ceph-mon[46969]: pgmap v232: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 730 op/s 2024-04-06T14:13:01.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:00 smithi134 ceph-mon[47549]: pgmap v233: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 736 op/s 2024-04-06T14:13:01.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:00 smithi105 ceph-mon[46969]: pgmap v233: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 736 op/s 2024-04-06T14:13:01.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:00 smithi090 ceph-mon[43420]: pgmap v233: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 158 KiB/s rd, 15 MiB/s wr, 736 op/s 2024-04-06T14:13:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 435.0 s, 1504.4 tps, lat 66.543 ms stddev 121.008 2024-04-06T14:13:03.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:02 smithi134 ceph-mon[47549]: pgmap v234: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 177 KiB/s rd, 21 MiB/s wr, 1.09k op/s 2024-04-06T14:13:03.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:02 smithi105 ceph-mon[46969]: pgmap v234: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 177 KiB/s rd, 21 MiB/s wr, 1.09k op/s 2024-04-06T14:13:03.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:02 smithi090 ceph-mon[43420]: pgmap v234: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 177 KiB/s rd, 21 MiB/s wr, 1.09k op/s 2024-04-06T14:13:05.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:04 smithi134 ceph-mon[47549]: pgmap v235: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 14 MiB/s wr, 740 op/s 2024-04-06T14:13:05.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:04 smithi105 ceph-mon[46969]: pgmap v235: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 14 MiB/s wr, 740 op/s 2024-04-06T14:13:05.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:04 smithi090 ceph-mon[43420]: pgmap v235: 97 pgs: 97 active+clean; 8.2 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 14 MiB/s wr, 740 op/s 2024-04-06T14:13:07.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:06 smithi134 ceph-mon[47549]: pgmap v236: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 30 MiB/s wr, 1.99k op/s 2024-04-06T14:13:07.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:06 smithi105 ceph-mon[46969]: pgmap v236: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 30 MiB/s wr, 1.99k op/s 2024-04-06T14:13:07.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:06 smithi090 ceph-mon[43420]: pgmap v236: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 30 MiB/s wr, 1.99k op/s 2024-04-06T14:13:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 440.0 s, 1936.0 tps, lat 51.517 ms stddev 58.884 2024-04-06T14:13:09.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:08 smithi090 ceph-mon[43420]: pgmap v237: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.62k op/s 2024-04-06T14:13:09.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:08 smithi134 ceph-mon[47549]: pgmap v237: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.62k op/s 2024-04-06T14:13:09.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:08 smithi105 ceph-mon[46969]: pgmap v237: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.62k op/s 2024-04-06T14:13:11.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:10 smithi134 ceph-mon[47549]: pgmap v238: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.63k op/s 2024-04-06T14:13:11.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:10 smithi105 ceph-mon[46969]: pgmap v238: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.63k op/s 2024-04-06T14:13:11.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:10 smithi090 ceph-mon[43420]: pgmap v238: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 23 MiB/s wr, 1.63k op/s 2024-04-06T14:13:12.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:11 smithi134 ceph-mon[47549]: osdmap e87: 12 total, 12 up, 12 in 2024-04-06T14:13:12.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:11 smithi134 ceph-mon[47549]: pgmap v240: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:12.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:11 smithi105 ceph-mon[46969]: osdmap e87: 12 total, 12 up, 12 in 2024-04-06T14:13:12.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:11 smithi105 ceph-mon[46969]: pgmap v240: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:12.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:11 smithi090 ceph-mon[43420]: osdmap e87: 12 total, 12 up, 12 in 2024-04-06T14:13:12.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:11 smithi090 ceph-mon[43420]: pgmap v240: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:13.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 445.0 s, 1409.0 tps, lat 70.366 ms stddev 99.017 2024-04-06T14:13:15.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:14 smithi134 ceph-mon[47549]: pgmap v241: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:15.182 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:13:15.182 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:13:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:14 smithi105 ceph-mon[46969]: pgmap v241: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:15.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:14 smithi090 ceph-mon[43420]: pgmap v241: 97 pgs: 97 active+clean; 8.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 93 KiB/s rd, 25 MiB/s wr, 1.83k op/s 2024-04-06T14:13:15.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 -- 172.21.15.90:0/1975064721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 msgr2=0x7fbc2810cad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/1975064721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2810cad0 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fbc180099c0 tx=0x7fbc18009cd0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 -- 172.21.15.90:0/1975064721 shutdown_connections 2024-04-06T14:13:15.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/1975064721 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2810cad0 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/1975064721 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28105da0 0x7fbc28106180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/1975064721 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc28100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.531+0000 7fbc2ec46700 1 -- 172.21.15.90:0/1975064721 >> 172.21.15.90:0/1975064721 conn(0x7fbc280769b0 msgr2=0x7fbc28076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.532+0000 7fbc2ec46700 1 -- 172.21.15.90:0/1975064721 shutdown_connections 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.532+0000 7fbc2ec46700 1 -- 172.21.15.90:0/1975064721 wait complete. 2024-04-06T14:13:15.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 Processor -- start 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 -- start start 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc2819be10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2819c390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 0x7fbc28195430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc28195c00 con 0x7fbc28194f80 2024-04-06T14:13:15.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc28195d70 con 0x7fbc280fff00 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.533+0000 7fbc2ec46700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc28199a20 con 0x7fbc281008c0 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc2d1e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 0x7fbc28195430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc2d1e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 0x7fbc28195430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41012/0 (socket says 172.21.15.90:41012) 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc2d1e3700 1 -- 172.21.15.90:0/833301186 learned_addr learned my addr 172.21.15.90:0/833301186 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2819c390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc2819be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 -- 172.21.15.90:0/833301186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 msgr2=0x7fbc2819be10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc2819be10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 -- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 msgr2=0x7fbc28195430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 0x7fbc28195430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc27fff700 1 -- 172.21.15.90:0/833301186 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc180095f0 con 0x7fbc281008c0 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.534+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc2819be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:13:15.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc27fff700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2819c390 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fbc20007ec0 tx=0x7fbc2000d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc25ffb700 1 -- 172.21.15.90:0/833301186 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc2000ddd0 con 0x7fbc281008c0 2024-04-06T14:13:15.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc28199d00 con 0x7fbc281008c0 2024-04-06T14:13:15.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc2819a260 con 0x7fbc281008c0 2024-04-06T14:13:15.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc25ffb700 1 -- 172.21.15.90:0/833301186 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc2000f040 con 0x7fbc281008c0 2024-04-06T14:13:15.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.535+0000 7fbc25ffb700 1 -- 172.21.15.90:0/833301186 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc20005a40 con 0x7fbc281008c0 2024-04-06T14:13:15.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.539+0000 7fbc25ffb700 1 -- 172.21.15.90:0/833301186 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fbc20015400 con 0x7fbc281008c0 2024-04-06T14:13:15.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.540+0000 7fbc3adfab80 1 Processor -- start 2024-04-06T14:13:15.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.540+0000 7fbc25ffb700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 0x7fbc100a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.540+0000 7fbc25ffb700 1 -- 172.21.15.90:0/833301186 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fbc200cc590 con 0x7fbc281008c0 2024-04-06T14:13:15.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 -- start start 2024-04-06T14:13:15.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e80b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e8d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14dd8990 con 0x561f14e810a0 2024-04-06T14:13:15.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14dd6350 con 0x561f14e80780 2024-04-06T14:13:15.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.541+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14dd1c40 con 0x561f14e8ade0 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41016/0 (socket says 172.21.15.90:41016) 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 -- 172.21.15.90:0/515095487 learned_addr learned my addr 172.21.15.90:0/515095487 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e8d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 0x7fbc100a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 -- 172.21.15.90:0/515095487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 msgr2=0x561f14e8d1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e8d1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 -- 172.21.15.90:0/515095487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 msgr2=0x561f14e80b60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e80b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.542+0000 7fbc27fff700 1 -- 172.21.15.90:0/515095487 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561f14c533b0 con 0x561f14e810a0 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.543+0000 7fbc27fff700 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fbc200095b0 tx=0x7fbc200994d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9d99da72494429cd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.543+0000 7fbc0effd700 1 -- 172.21.15.90:0/515095487 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc200b90c0 con 0x561f14e810a0 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.543+0000 7fbc0effd700 1 -- 172.21.15.90:0/515095487 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc2009d420 con 0x561f14e810a0 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.543+0000 7fbc0effd700 1 -- 172.21.15.90:0/515095487 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc200ca0d0 con 0x561f14e810a0 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.544+0000 7fbc3adfab80 1 -- 172.21.15.90:0/515095487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 msgr2=0x561f14e8a8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.544+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 secure :-1 s=READY pgs=642 cs=0 l=1 rev1=1 crypto rx=0x7fbc200095b0 tx=0x7fbc200994d0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.545+0000 7fbc3adfab80 1 -- 172.21.15.90:0/515095487 shutdown_connections 2024-04-06T14:13:15.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.545+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e8d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.545+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e80b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.545+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/515095487 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e8a8a0 unknown :-1 s=CLOSED pgs=642 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.545+0000 7fbc3adfab80 1 -- 172.21.15.90:0/515095487 >> 172.21.15.90:0/515095487 conn(0x561f14ddf6a0 msgr2=0x561f14e7ba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.546+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 0x7fbc100a3830 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x561f14e821f0 tx=0x7fbc180058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.548+0000 7fbc3adfab80 1 -- 172.21.15.90:0/515095487 shutdown_connections 2024-04-06T14:13:15.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.548+0000 7fbc3adfab80 1 -- 172.21.15.90:0/515095487 wait complete. 2024-04-06T14:13:15.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.548+0000 7fbc3adfab80 1 -- start start 2024-04-06T14:13:15.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.548+0000 7fbc3adfab80 1 Processor -- start 2024-04-06T14:13:15.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e87a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e88070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14d6dd80 con 0x561f14e810a0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14d6da20 con 0x561f14e80780 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14c556a0 con 0x561f14e8ade0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc3adfab80 10 client.0 _ll_register_callbacks cb 0x561f14d77470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc2c9e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc2c9e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42856/0 (socket says 172.21.15.90:42856) 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.549+0000 7fbc2c9e2700 1 -- 172.21.15.90:0/611589497 learned_addr learned my addr 172.21.15.90:0/611589497 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e88070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc27fff700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e87a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 -- 172.21.15.90:0/611589497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 msgr2=0x561f14e88070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e88070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 -- 172.21.15.90:0/611589497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 msgr2=0x561f14e87a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e87a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 -- 172.21.15.90:0/611589497 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc2819a260 con 0x561f14e80780 2024-04-06T14:13:15.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.550+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fbc1c00d160 tx=0x7fbc1c00d520 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6490809f03983a5c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.551+0000 7fbc07fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:15.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.551+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc1c00dda0 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.551+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fbc1c013480 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x561f14d92380 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbc1c013640 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 10 client.26411 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561f14bffb60 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x561f14d92380 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbc1c012970 con 0x561f14e80780 2024-04-06T14:13:15.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 10 client.26411 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 10 client.26411 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 20 client.26411 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14e9aac0 0x561f14e9cee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 4 client.26411 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.552+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561f14dca700 con 0x561f14e9aac0 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.553+0000 7fbc27fff700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14e9aac0 0x561f14e9cee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.553+0000 7fbc07fff700 10 client.26411 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:15.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.553+0000 7fbc27fff700 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14e9aac0 0x561f14e9cee0 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=17a100a6b7dfff2a server_cookie=2f4aa4c65de5e8b0 in_seq=0 out_seq=0 2024-04-06T14:13:15.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc07fff700 1 -- 172.21.15.90:0/611589497 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x561f14dca700 con 0x561f14e9aac0 2024-04-06T14:13:15.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc07fff700 10 client.26411 handle_command_reply: tid=0 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 client.26411 shutdown 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 msgr2=0x561f14e8a770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fbc1c00d160 tx=0x7fbc1c00d520 comp rx=0 tx=0).stop 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 shutdown_connections 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14e9aac0 0x561f14e9cee0 crc :-1 s=READY pgs=145 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e88070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e8a770 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/611589497 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e810a0 0x561f14e87a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 >> 172.21.15.90:0/611589497 conn(0x561f14ddf6a0 msgr2=0x561f14e7ac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc07fff700 0 client.26411 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:15.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 shutdown_connections 2024-04-06T14:13:15.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.554+0000 7fbc3adfab80 1 -- 172.21.15.90:0/611589497 wait complete. 2024-04-06T14:13:15.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.555+0000 7fbc3adfab80 20 client.26411 trim_cache size 0 max 16384 2024-04-06T14:13:15.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.585+0000 7fbc3adfab80 1 Processor -- start 2024-04-06T14:13:15.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 -- start start 2024-04-06T14:13:15.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e91ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e8ade0 0x561f14e86870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14e9d270 con 0x561f14e8ade0 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14e86db0 con 0x561f14e80780 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14e87830 con 0x561f14e810a0 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc27fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc27fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33834/0 (socket says 172.21.15.90:33834) 2024-04-06T14:13:15.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.586+0000 7fbc27fff700 1 -- 172.21.15.90:0/2477673645 learned_addr learned my addr 172.21.15.90:0/2477673645 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.594+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e8ade0 0x561f14e86870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.599+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e91ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.602+0000 7fbc27fff700 1 -- 172.21.15.90:0/2477673645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 msgr2=0x561f14e91ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.602+0000 7fbc27fff700 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e91ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.602+0000 7fbc27fff700 1 -- 172.21.15.90:0/2477673645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e8ade0 msgr2=0x561f14e86870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.602+0000 7fbc27fff700 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e8ade0 0x561f14e86870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.602+0000 7fbc27fff700 1 -- 172.21.15.90:0/2477673645 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561f14d97ea0 con 0x561f14e810a0 2024-04-06T14:13:15.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.613+0000 7fbc27fff700 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009440 tx=0x7fbc20006040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c5a47dfece090249 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.614+0000 7fbc0f7fe700 1 -- 172.21.15.90:0/2477673645 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc200b9070 con 0x561f14e810a0 2024-04-06T14:13:15.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.614+0000 7fbc0f7fe700 1 -- 172.21.15.90:0/2477673645 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc200b7900 con 0x561f14e810a0 2024-04-06T14:13:15.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.614+0000 7fbc0f7fe700 1 -- 172.21.15.90:0/2477673645 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc200c9eb0 con 0x561f14e810a0 2024-04-06T14:13:15.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.615+0000 7fbc3adfab80 1 -- 172.21.15.90:0/2477673645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 msgr2=0x561f14e86330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.615+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fbc20009440 tx=0x7fbc20006040 comp rx=0 tx=0).stop 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 -- 172.21.15.90:0/2477673645 shutdown_connections 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e8ade0 0x561f14e86870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e80780 0x561f14e91ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/2477673645 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e810a0 0x561f14e86330 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 -- 172.21.15.90:0/2477673645 >> 172.21.15.90:0/2477673645 conn(0x561f14ddf6a0 msgr2=0x561f14e82d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 -- 172.21.15.90:0/2477673645 shutdown_connections 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 -- 172.21.15.90:0/2477673645 wait complete. 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 -- start start 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.620+0000 7fbc3adfab80 1 Processor -- start 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e80780 0x561f14e90030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e810a0 0x561f14e90570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14eee8c0 con 0x561f14e80780 2024-04-06T14:13:15.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14eeea00 con 0x561f14e810a0 2024-04-06T14:13:15.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561f14eeeb40 con 0x561f14e8ade0 2024-04-06T14:13:15.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc3adfab80 10 client.0 _ll_register_callbacks cb 0x561f14d77470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc2d1e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc2d1e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33838/0 (socket says 172.21.15.90:33838) 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc2d1e3700 1 -- 172.21.15.90:0/3296666840 learned_addr learned my addr 172.21.15.90:0/3296666840 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.621+0000 7fbc27fff700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e810a0 0x561f14e90570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 -- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e810a0 msgr2=0x561f14e90570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e810a0 0x561f14e90570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 -- 172.21.15.90:0/3296666840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e80780 msgr2=0x561f14e90030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e80780 0x561f14e90030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 -- 172.21.15.90:0/3296666840 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x561f14c533b0 con 0x561f14e8ade0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc2d1e3700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fbc18005950 tx=0x7fbc18009cd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9eb4382198d40e2a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc07fff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:13:15.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc1802e040 con 0x561f14e8ade0 2024-04-06T14:13:15.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.622+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbc2009d420 con 0x561f14e8ade0 2024-04-06T14:13:15.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.623+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fbc1800fd60 con 0x561f14e8ade0 2024-04-06T14:13:15.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.623+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbc1800bee0 con 0x561f14e8ade0 2024-04-06T14:13:15.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.627+0000 7fbc3adfab80 10 client.26248 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:15.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.627+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561f14c57cd0 con 0x561f14e8ade0 2024-04-06T14:13:15.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.634+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbc2009d420 con 0x561f14e8ade0 2024-04-06T14:13:15.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.634+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbc18031030 con 0x561f14e8ade0 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 10 client.26248 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 10 client.26248 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 20 client.26248 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14ef0310 0x561f14ef2730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 4 client.26248 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:13:15.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.636+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x561f14c20cf0 con 0x561f14ef0310 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.640+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14ef0310 0x561f14ef2730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.640+0000 7fbc2c9e2700 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14ef0310 0x561f14ef2730 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=623c03f59b617279 server_cookie=fb540f3daa1cf5dc in_seq=0 out_seq=0 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc07fff700 10 client.26248 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc07fff700 1 -- 172.21.15.90:0/3296666840 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x561f14c20cf0 con 0x561f14ef0310 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc07fff700 10 client.26248 handle_command_reply: tid=0 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 client.26248 shutdown 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 msgr2=0x561f14e90ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7fbc18005950 tx=0x7fbc18009cd0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 shutdown_connections 2024-04-06T14:13:15.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x561f14ef0310 0x561f14ef2730 crc :-1 s=READY pgs=146 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561f14e8ade0 0x561f14e90ab0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc07fff700 0 client.26248 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561f14e80780 0x561f14e90030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 --2- 172.21.15.90:0/3296666840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561f14e810a0 0x561f14e90570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 >> 172.21.15.90:0/3296666840 conn(0x561f14ddf6a0 msgr2=0x561f14e82d60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 shutdown_connections 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 1 -- 172.21.15.90:0/3296666840 wait complete. 2024-04-06T14:13:15.646 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.643+0000 7fbc3adfab80 20 client.26248 trim_cache size 0 max 16384 2024-04-06T14:13:15.647 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:15.647 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:13:15.647 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:13:15.647 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 msgr2=0x7fbc100a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 0x7fbc100a3830 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x561f14e821f0 tx=0x7fbc180058e0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 msgr2=0x7fbc2819c390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2819c390 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fbc20007ec0 tx=0x7fbc2000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 shutdown_connections 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbc100a1370 0x7fbc100a3830 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbc28194f80 0x7fbc28195430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc280fff00 0x7fbc2819be10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 --2- 172.21.15.90:0/833301186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbc281008c0 0x7fbc2819c390 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 >> 172.21.15.90:0/833301186 conn(0x7fbc280769b0 msgr2=0x7fbc280fec30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 shutdown_connections 2024-04-06T14:13:15.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.646+0000 7fbc2ec46700 1 -- 172.21.15.90:0/833301186 wait complete. 2024-04-06T14:13:15.668 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:13:15.668 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:13:15.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 -- 172.21.15.90:0/200466409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d010d330 msgr2=0x7f46d010d710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 --2- 172.21.15.90:0/200466409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d010d330 0x7f46d010d710 secure :-1 s=READY pgs=643 cs=0 l=1 rev1=1 crypto rx=0x7f46c000b0a0 tx=0x7f46c000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 -- 172.21.15.90:0/200466409 shutdown_connections 2024-04-06T14:13:15.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 --2- 172.21.15.90:0/200466409 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d0108770 0x7f46d0078fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 --2- 172.21.15.90:0/200466409 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d010d330 0x7f46d010d710 unknown :-1 s=CLOSED pgs=643 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 --2- 172.21.15.90:0/200466409 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46d0107db0 0x7f46d0108230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.863+0000 7f46d6685700 1 -- 172.21.15.90:0/200466409 >> 172.21.15.90:0/200466409 conn(0x7f46d006bd30 msgr2=0x7f46d006c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:15.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.866+0000 7f46d6685700 1 -- 172.21.15.90:0/200466409 shutdown_connections 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.870+0000 7f46d6685700 1 -- 172.21.15.90:0/200466409 wait complete. 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.870+0000 7f46d6685700 1 Processor -- start 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.870+0000 7f46d6685700 1 -- start start 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46d0107db0 0x7f46d007e390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d0108770 0x7f46d007e8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46d01bb3e0 con 0x7f46d0108770 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46d01bb520 con 0x7f46d0107db0 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d6685700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46d01bb6e0 con 0x7f46d010d330 2024-04-06T14:13:15.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d4c22700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d4c22700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33852/0 (socket says 172.21.15.90:33852) 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.871+0000 7f46d4c22700 1 -- 172.21.15.90:0/3734167245 learned_addr learned my addr 172.21.15.90:0/3734167245 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.872+0000 7f46d4c22700 1 -- 172.21.15.90:0/3734167245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46d0107db0 msgr2=0x7f46d007e390 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.872+0000 7f46d4c22700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46d0107db0 0x7f46d007e390 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.872+0000 7f46d4c22700 1 -- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d0108770 msgr2=0x7f46d007e8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.872+0000 7f46d4c22700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d0108770 0x7f46d007e8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:15.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.872+0000 7f46d4c22700 1 -- 172.21.15.90:0/3734167245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f46c0009d00 con 0x7f46d010d330 2024-04-06T14:13:15.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46d4c22700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f46c800bfa0 tx=0x7f46c800de50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46c8008a60 con 0x7f46d010d330 2024-04-06T14:13:15.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f46d007a110 con 0x7f46d010d330 2024-04-06T14:13:15.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46d007a670 con 0x7f46d010d330 2024-04-06T14:13:15.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f46c8008bc0 con 0x7f46d010d330 2024-04-06T14:13:15.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.873+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f46c801a570 con 0x7f46d010d330 2024-04-06T14:13:15.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.877+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f46c8012450 con 0x7f46d010d330 2024-04-06T14:13:15.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.877+0000 7f46cd7fa700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 0x7f46bc0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:15.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.878+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f46c80cfdf0 con 0x7f46d010d330 2024-04-06T14:13:15.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.878+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f46b00052f0 con 0x7f46d010d330 2024-04-06T14:13:15.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.879+0000 7f46cffff700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 0x7f46bc0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:15.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.880+0000 7f46cffff700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 0x7f46bc0a3890 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f46c0000f80 tx=0x7f46c0005a60 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:15.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:15.886+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f46c80991b0 con 0x7f46d010d330 2024-04-06T14:13:16.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.284+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f46b00055a0 con 0x7f46d010d330 2024-04-06T14:13:16.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.291+0000 7f46cd7fa700 1 -- 172.21.15.90:0/3734167245 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f46c8021020 con 0x7f46d010d330 2024-04-06T14:13:16.292 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:13:16.292 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:13:16.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.295+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 msgr2=0x7f46bc0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.295+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 0x7f46bc0a3890 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f46c0000f80 tx=0x7f46c0005a60 comp rx=0 tx=0).stop 2024-04-06T14:13:16.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.295+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 msgr2=0x7f46d01bae10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.295+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f46c800bfa0 tx=0x7f46c800de50 comp rx=0 tx=0).stop 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 shutdown_connections 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46bc0a13d0 0x7f46bc0a3890 secure :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f46c0000f80 tx=0x7f46c0005a60 comp rx=0 tx=0).stop 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46d010d330 0x7f46d01bae10 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46d0107db0 0x7f46d007e390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 --2- 172.21.15.90:0/3734167245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f46d0108770 0x7f46d007e8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.298+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 >> 172.21.15.90:0/3734167245 conn(0x7f46d006bd30 msgr2=0x7f46d0071960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.305+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 shutdown_connections 2024-04-06T14:13:16.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.306+0000 7f46d6685700 1 -- 172.21.15.90:0/3734167245 wait complete. 2024-04-06T14:13:16.313 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:13:16.333 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:16.333 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.530+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2010307610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4069be0 msgr2=0x7f66a4105980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.530+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2010307610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4069be0 0x7f66a4105980 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f6698009990 tx=0x7f6698009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.531+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2010307610 shutdown_connections 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.531+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2010307610 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a41060b0 0x7f66a410dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.531+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2010307610 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a4069230 0x7f66a4069610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.531+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2010307610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a4069be0 0x7f66a4105980 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.531+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2010307610 >> 172.21.15.90:0/2010307610 conn(0x7f66a40769b0 msgr2=0x7f66a4076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.537+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2010307610 shutdown_connections 2024-04-06T14:13:16.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.539+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2010307610 wait complete. 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.539+0000 7f66a9d4b700 1 Processor -- start 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 -- start start 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4069230 0x7f66a41a33b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a410b980 0x7f66a410be00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a410c3d0 con 0x7f66a41060b0 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a410c540 con 0x7f66a410b980 2024-04-06T14:13:16.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a9d4b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f66a410c700 con 0x7f66a4069230 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a410b980 0x7f66a410be00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41066/0 (socket says 172.21.15.90:41066) 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 -- 172.21.15.90:0/2310563449 learned_addr learned my addr 172.21.15.90:0/2310563449 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4069230 0x7f66a41a33b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 -- 172.21.15.90:0/2310563449 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4069230 msgr2=0x7f66a41a33b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4069230 0x7f66a41a33b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 -- 172.21.15.90:0/2310563449 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a410b980 msgr2=0x7f66a410be00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a410b980 0x7f66a410be00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.540+0000 7f66a2ffd700 1 -- 172.21.15.90:0/2310563449 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66980095f0 con 0x7f66a41060b0 2024-04-06T14:13:16.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7f669800bf30 tx=0x7f669800bf60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a0ff9700 1 -- 172.21.15.90:0/2310563449 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669801d070 con 0x7f66a41060b0 2024-04-06T14:13:16.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f66a41aa270 con 0x7f66a41060b0 2024-04-06T14:13:16.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a0ff9700 1 -- 172.21.15.90:0/2310563449 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6698004210 con 0x7f66a41060b0 2024-04-06T14:13:16.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a0ff9700 1 -- 172.21.15.90:0/2310563449 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6698021740 con 0x7f66a41060b0 2024-04-06T14:13:16.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.541+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f66a41aa7a0 con 0x7f66a41060b0 2024-04-06T14:13:16.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.546+0000 7f66a0ff9700 1 -- 172.21.15.90:0/2310563449 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f669800b880 con 0x7f66a41060b0 2024-04-06T14:13:16.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.547+0000 7f66a0ff9700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 0x7f668c0a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.547+0000 7f66a0ff9700 1 -- 172.21.15.90:0/2310563449 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f66980cdfc0 con 0x7f66a41060b0 2024-04-06T14:13:16.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 0x7f668c0a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 Processor -- start 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 -- start start 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc3178a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc31a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc265990 con 0x559acc317de0 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc263350 con 0x559acc30e0a0 2024-04-06T14:13:16.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66b5effb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc25ec40 con 0x559acc30d780 2024-04-06T14:13:16.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.548+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33878/0 (socket says 172.21.15.90:33878) 2024-04-06T14:13:16.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 -- 172.21.15.90:0/1642505058 learned_addr learned my addr 172.21.15.90:0/1642505058 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 0x7f668c0a3790 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f669400b450 tx=0x7f669400d040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 -- 172.21.15.90:0/1642505058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 msgr2=0x559acc3178a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc3178a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 -- 172.21.15.90:0/1642505058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 msgr2=0x559acc31a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc31a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 -- 172.21.15.90:0/1642505058 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559acc0e03b0 con 0x559acc30d780 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f66a37fe700 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f6694016730 tx=0x7f6694011000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4a78c64cd463ca99 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.549+0000 7f6689ffb700 1 -- 172.21.15.90:0/1642505058 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6694010040 con 0x559acc30d780 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.550+0000 7f6689ffb700 1 -- 172.21.15.90:0/1642505058 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f669401c040 con 0x559acc30d780 2024-04-06T14:13:16.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.550+0000 7f6689ffb700 1 -- 172.21.15.90:0/1642505058 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6694020b30 con 0x559acc30d780 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.552+0000 7f66b5effb80 1 -- 172.21.15.90:0/1642505058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 msgr2=0x559acc30db60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.552+0000 7f66b5effb80 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f6694016730 tx=0x7f6694011000 comp rx=0 tx=0).stop 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 -- 172.21.15.90:0/1642505058 shutdown_connections 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc31a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc30db60 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 --2- 172.21.15.90:0/1642505058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc3178a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 -- 172.21.15.90:0/1642505058 >> 172.21.15.90:0/1642505058 conn(0x559acc26c6a0 msgr2=0x559acc308a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 -- 172.21.15.90:0/1642505058 shutdown_connections 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 -- 172.21.15.90:0/1642505058 wait complete. 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.553+0000 7f66b5effb80 1 -- start start 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 Processor -- start 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc317de0 0x559acc315350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc1fad80 con 0x559acc30d780 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc1faa20 con 0x559acc30e0a0 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc0e26a0 con 0x559acc317de0 2024-04-06T14:13:16.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.554+0000 7f66b5effb80 10 client.0 _ll_register_callbacks cb 0x559acc204470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:16.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41074/0 (socket says 172.21.15.90:41074) 2024-04-06T14:13:16.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 -- 172.21.15.90:0/2770781702 learned_addr learned my addr 172.21.15.90:0/2770781702 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 -- 172.21.15.90:0/2770781702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc317de0 msgr2=0x559acc315350 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:16.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc317de0 0x559acc315350 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 -- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 msgr2=0x559acc314c10 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:16.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314c10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.556+0000 7f66a37fe700 1 -- 172.21.15.90:0/2770781702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6694016410 con 0x559acc30d780 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.557+0000 7f66a37fe700 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7f6694020430 tx=0x7f6694021f40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=935f2ea3ce63ca1e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.557+0000 7f6682ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.557+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6694010040 con 0x559acc30d780 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.557+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f669400aa00 con 0x559acc30d780 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.557+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559acc21f380 con 0x559acc30d780 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f669400abe0 con 0x559acc30d780 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 10 client.16695 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:16.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559acc08cb60 con 0x559acc30d780 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559acc21f380 con 0x559acc30d780 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f669401f070 con 0x559acc30d780 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 10 client.16695 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 10 client.16695 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 20 client.16695 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:16.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc338880 0x559acc33ad20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 4 client.16695 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:16.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.558+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559acc257700 con 0x559acc338880 2024-04-06T14:13:16.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.559+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc338880 0x559acc33ad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.559+0000 7f6682ffd700 10 client.16695 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.559+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc338880 0x559acc33ad20 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=16ee134774af8257 server_cookie=bdf8092ad666777b in_seq=0 out_seq=0 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.560+0000 7f6682ffd700 1 -- 172.21.15.90:0/2770781702 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x559acc257700 con 0x559acc338880 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.560+0000 7f6682ffd700 10 client.16695 handle_command_reply: tid=0 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.560+0000 7f66b5effb80 1 client.16695 shutdown 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.560+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 msgr2=0x559acc317880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.560+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 secure :-1 s=READY pgs=645 cs=0 l=1 rev1=1 crypto rx=0x7f6694020430 tx=0x7f6694021f40 comp rx=0 tx=0).stop 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 shutdown_connections 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc338880 0x559acc33ad20 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc317de0 0x559acc315350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc30d780 0x559acc317880 unknown :-1 s=CLOSED pgs=645 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2770781702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 >> 172.21.15.90:0/2770781702 conn(0x559acc26c6a0 msgr2=0x559acc307c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f6682ffd700 0 client.16695 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:16.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 shutdown_connections 2024-04-06T14:13:16.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 1 -- 172.21.15.90:0/2770781702 wait complete. 2024-04-06T14:13:16.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.568+0000 7f66b5effb80 20 client.16695 trim_cache size 0 max 16384 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 Processor -- start 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 -- start start 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc3144f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc30e480 con 0x559acc317de0 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc3003c0 con 0x559acc30e0a0 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.589+0000 7f66b5effb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc313160 con 0x559acc30d780 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41088/0 (socket says 172.21.15.90:41088) 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 -- 172.21.15.90:0/2969057828 learned_addr learned my addr 172.21.15.90:0/2969057828 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 -- 172.21.15.90:0/2969057828 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 msgr2=0x559acc3144f0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:16.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc3144f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 -- 172.21.15.90:0/2969057828 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 msgr2=0x559acc314a30 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:16.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314a30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.590+0000 7f66a3fff700 1 -- 172.21.15.90:0/2969057828 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559acc224ea0 con 0x559acc317de0 2024-04-06T14:13:16.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.591+0000 7f66a3fff700 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f669c009c90 tx=0x7f669c033450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c26ed13de91e4c34 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.595+0000 7f668a7fc700 1 -- 172.21.15.90:0/2969057828 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669c033ca0 con 0x559acc317de0 2024-04-06T14:13:16.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.595+0000 7f668a7fc700 1 -- 172.21.15.90:0/2969057828 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f669c033e00 con 0x559acc317de0 2024-04-06T14:13:16.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.595+0000 7f668a7fc700 1 -- 172.21.15.90:0/2969057828 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f669c03b420 con 0x559acc317de0 2024-04-06T14:13:16.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.602+0000 7f66b5effb80 1 -- 172.21.15.90:0/2969057828 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 msgr2=0x559acc3083a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.602+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 secure :-1 s=READY pgs=646 cs=0 l=1 rev1=1 crypto rx=0x7f669c009c90 tx=0x7f669c033450 comp rx=0 tx=0).stop 2024-04-06T14:13:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.614+0000 7f66b5effb80 1 -- 172.21.15.90:0/2969057828 shutdown_connections 2024-04-06T14:13:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.614+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc3083a0 unknown :-1 s=CLOSED pgs=646 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.614+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30d780 0x559acc3144f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.614+0000 7f66b5effb80 1 --2- 172.21.15.90:0/2969057828 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30e0a0 0x559acc314a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.614+0000 7f66b5effb80 1 -- 172.21.15.90:0/2969057828 >> 172.21.15.90:0/2969057828 conn(0x559acc26c6a0 msgr2=0x559acc33a890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.623+0000 7f66b5effb80 1 -- 172.21.15.90:0/2969057828 shutdown_connections 2024-04-06T14:13:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.626+0000 7f66b5effb80 1 -- 172.21.15.90:0/2969057828 wait complete. 2024-04-06T14:13:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.626+0000 7f66b5effb80 1 -- start start 2024-04-06T14:13:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.626+0000 7f66b5effb80 1 Processor -- start 2024-04-06T14:13:16.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30e0a0 0x559acc32df00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc32e4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc32ebb0 con 0x559acc317de0 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc32ed20 con 0x559acc30d780 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559acc38c780 con 0x559acc30e0a0 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66b5effb80 10 client.0 _ll_register_callbacks cb 0x559acc204470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66a2ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30e0a0 0x559acc32df00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66a37fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42908/0 (socket says 172.21.15.90:42908) 2024-04-06T14:13:16.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.627+0000 7f66a37fe700 1 -- 172.21.15.90:0/923347115 learned_addr learned my addr 172.21.15.90:0/923347115 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 -- 172.21.15.90:0/923347115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30e0a0 msgr2=0x559acc32df00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30e0a0 0x559acc32df00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 -- 172.21.15.90:0/923347115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 msgr2=0x559acc32e4a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc32e4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 -- 172.21.15.90:0/923347115 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f669c009970 con 0x559acc30d780 2024-04-06T14:13:16.629 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.628+0000 7f66a37fe700 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f6694000f80 tx=0x7f6694029740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ba48cc6599a613c2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.630+0000 7f6682ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:16.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.630+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6694010070 con 0x559acc30d780 2024-04-06T14:13:16.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.630+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f6694032a60 con 0x559acc30d780 2024-04-06T14:13:16.633 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.633+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f669401c040 con 0x559acc30d780 2024-04-06T14:13:16.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.634+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6694032c20 con 0x559acc30d780 2024-04-06T14:13:16.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.638+0000 7f66b5effb80 10 client.26426 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:16.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.638+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559acc0e4cd0 con 0x559acc30d780 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.650+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f669401c040 con 0x559acc30d780 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.650+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6694016af0 con 0x559acc30d780 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 10 client.26426 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 10 client.26426 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 20 client.26426 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc38da90 0x559acc38feb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 4 client.26426 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:16.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.651+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x559acc0adcf0 con 0x559acc38da90 2024-04-06T14:13:16.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.655+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc38da90 0x559acc38feb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f6682ffd700 10 client.26426 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f66a2ffd700 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc38da90 0x559acc38feb0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=fe8e2dccd68cfb81 server_cookie=29b9d10ba5affdec in_seq=0 out_seq=0 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f6682ffd700 1 -- 172.21.15.90:0/923347115 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x559acc0adcf0 con 0x559acc38da90 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f6682ffd700 10 client.26426 handle_command_reply: tid=0 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f66b5effb80 1 client.26426 shutdown 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 msgr2=0x559acc32fac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.656+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f6694000f80 tx=0x7f6694029740 comp rx=0 tx=0).stop 2024-04-06T14:13:16.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 shutdown_connections 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x559acc38da90 0x559acc38feb0 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559acc317de0 0x559acc32e4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559acc30d780 0x559acc32fac0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 --2- 172.21.15.90:0/923347115 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559acc30e0a0 0x559acc32df00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 >> 172.21.15.90:0/923347115 conn(0x559acc26c6a0 msgr2=0x559acc3163a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f6682ffd700 0 client.26426 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 shutdown_connections 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 1 -- 172.21.15.90:0/923347115 wait complete. 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.660+0000 7f66b5effb80 20 client.26426 trim_cache size 0 max 16384 2024-04-06T14:13:16.661 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:16.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 msgr2=0x7f668c0a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 0x7f668c0a3790 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f669400b450 tx=0x7f669400d040 comp rx=0 tx=0).stop 2024-04-06T14:13:16.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 msgr2=0x7f66a41a38f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.663 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 secure :-1 s=READY pgs=644 cs=0 l=1 rev1=1 crypto rx=0x7f669800bf30 tx=0x7f669800bf60 comp rx=0 tx=0).stop 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 shutdown_connections 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f668c0a12d0 0x7f668c0a3790 secure :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f669400b450 tx=0x7f669400d040 comp rx=0 tx=0).stop 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f66a410b980 0x7f66a410be00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f66a4069230 0x7f66a41a33b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 --2- 172.21.15.90:0/2310563449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f66a41060b0 0x7f66a41a38f0 unknown :-1 s=CLOSED pgs=644 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.664 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.663+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 >> 172.21.15.90:0/2310563449 conn(0x7f66a40769b0 msgr2=0x7f66a4104a60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.666+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 shutdown_connections 2024-04-06T14:13:16.667 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.666+0000 7f66a9d4b700 1 -- 172.21.15.90:0/2310563449 wait complete. 2024-04-06T14:13:16.686 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:16.687 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 -- 172.21.15.90:0/2959423006 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 msgr2=0x7f9bf81162f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/2959423006 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81162f0 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f9be0009990 tx=0x7f9be0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 -- 172.21.15.90:0/2959423006 shutdown_connections 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/2959423006 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81162f0 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/2959423006 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf8107db0 0x7f9bf8108190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/2959423006 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf81086d0 0x7f9bf81139c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.897+0000 7f9bfeb90700 1 -- 172.21.15.90:0/2959423006 >> 172.21.15.90:0/2959423006 conn(0x7f9bf806be00 msgr2=0x7f9bf806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.903+0000 7f9bfeb90700 1 -- 172.21.15.90:0/2959423006 shutdown_connections 2024-04-06T14:13:16.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.903+0000 7f9bfeb90700 1 -- 172.21.15.90:0/2959423006 wait complete. 2024-04-06T14:13:16.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.904+0000 7f9bfeb90700 1 Processor -- start 2024-04-06T14:13:16.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.904+0000 7f9bfeb90700 1 -- start start 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 0x7f9bf8112ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81aa970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bf81aafb0 con 0x7f9bf81086d0 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bf81ab120 con 0x7f9bf8113f00 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfeb90700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bf81ab2e0 con 0x7f9bf8107db0 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 0x7f9bf8112ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41130/0 (socket says 172.21.15.90:41130) 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bf7fff700 1 -- 172.21.15.90:0/3015003760 learned_addr learned my addr 172.21.15.90:0/3015003760 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfd12d700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81aa970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3015003760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 msgr2=0x7f9bf81aa970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81aa970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 msgr2=0x7f9bf81130e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3015003760 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9be00095f0 con 0x7f9bf8107db0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.905+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 0x7f9bf8112ba0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f9be800b450 tx=0x7f9be800b760 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.906+0000 7f9bf5ffb700 1 -- 172.21.15.90:0/3015003760 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9be8011620 con 0x7f9bf8107db0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.906+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bf81ab5c0 con 0x7f9bf8107db0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.906+0000 7f9bf5ffb700 1 -- 172.21.15.90:0/3015003760 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9be8011c40 con 0x7f9bf8107db0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.906+0000 7f9bf5ffb700 1 -- 172.21.15.90:0/3015003760 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9be8010d00 con 0x7f9bf8107db0 2024-04-06T14:13:16.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.906+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bf81b2e00 con 0x7f9bf8107db0 2024-04-06T14:13:16.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.908+0000 7f9bf5ffb700 1 -- 172.21.15.90:0/3015003760 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9be801a610 con 0x7f9bf8107db0 2024-04-06T14:13:16.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.909+0000 7f9bf5ffb700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 0x7f9be40a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.909+0000 7f9bf5ffb700 1 -- 172.21.15.90:0/3015003760 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f9be80cc790 con 0x7f9bf8107db0 2024-04-06T14:13:16.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.909+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 0x7f9be40a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 Processor -- start 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 -- start start 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281988eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5628197e6990 con 0x56281988e780 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5628197e4350 con 0x562819898de0 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5628197dfc40 con 0x56281988f0a0 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33930/0 (socket says 172.21.15.90:33930) 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9bf7fff700 1 -- 172.21.15.90:0/975024240 learned_addr learned my addr 172.21.15.90:0/975024240 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281988eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.910+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 0x7f9be40a3790 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f9bf81a7640 tx=0x7f9bf000b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 -- 172.21.15.90:0/975024240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 msgr2=0x56281989b1d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:16.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989b1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 -- 172.21.15.90:0/975024240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 msgr2=0x56281988eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281988eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 -- 172.21.15.90:0/975024240 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5628196613b0 con 0x56281988f0a0 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f9bf0015740 tx=0x7f9bf0011000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=48e722d359a4e55 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9beeffd700 1 -- 172.21.15.90:0/975024240 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bf001f5f0 con 0x56281988f0a0 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9beeffd700 1 -- 172.21.15.90:0/975024240 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bf000f040 con 0x56281988f0a0 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.911+0000 7f9beeffd700 1 -- 172.21.15.90:0/975024240 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bf00139b0 con 0x56281988f0a0 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.912+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/975024240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 msgr2=0x5628198988a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.912+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f9bf0015740 tx=0x7f9bf0011000 comp rx=0 tx=0).stop 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/975024240 shutdown_connections 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281988eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/975024240 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x5628198988a0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/975024240 >> 172.21.15.90:0/975024240 conn(0x5628197ed6a0 msgr2=0x562819889a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/975024240 shutdown_connections 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/975024240 wait complete. 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.913+0000 7f9c0ad44b80 1 -- start start 2024-04-06T14:13:16.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 Processor -- start 2024-04-06T14:13:16.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819895bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x562819896310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281977bd80 con 0x56281988e780 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281977ba20 con 0x562819898de0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5628196636a0 con 0x56281988f0a0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.914+0000 7f9c0ad44b80 10 client.0 _ll_register_callbacks cb 0x562819785470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41152/0 (socket says 172.21.15.90:41152) 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 -- 172.21.15.90:0/1099503874 learned_addr learned my addr 172.21.15.90:0/1099503874 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfd12d700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x562819896310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819895bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 -- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 msgr2=0x562819895bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819895bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 -- 172.21.15.90:0/1099503874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 msgr2=0x562819896310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x562819896310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.915+0000 7f9bfc92c700 1 -- 172.21.15.90:0/1099503874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bf00153f0 con 0x56281988e780 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7f9be8009fd0 tx=0x7f9be809a150 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44b1302486dbe51c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9bd7fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9be80a1d80 con 0x56281988e780 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f9be8010380 con 0x56281988e780 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5628197a0380 con 0x56281988e780 2024-04-06T14:13:16.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.916+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9be80b2200 con 0x56281988e780 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 10 client.16707 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56281960db60 con 0x56281988e780 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9be80b6070 con 0x56281988e780 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9be8098df0 con 0x56281988e780 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 10 client.16707 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 10 client.16707 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 20 client.16707 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5628198a8a40 0x5628198aae60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 4 client.16707 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5628197d8700 con 0x5628198a8a40 2024-04-06T14:13:16.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.919+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5628198a8a40 0x5628198aae60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.920+0000 7f9bd7fff700 10 client.16707 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.920+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5628198a8a40 0x5628198aae60 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=836d39fde241ce21 server_cookie=442601a425d4b1e8 in_seq=0 out_seq=0 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9bd7fff700 1 -- 172.21.15.90:0/1099503874 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5628197d8700 con 0x5628198a8a40 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9bd7fff700 10 client.16707 handle_command_reply: tid=0 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 client.16707 shutdown 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 msgr2=0x562819898840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 secure :-1 s=READY pgs=647 cs=0 l=1 rev1=1 crypto rx=0x7f9be8009fd0 tx=0x7f9be809a150 comp rx=0 tx=0).stop 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 shutdown_connections 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5628198a8a40 0x5628198aae60 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x562819896310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x562819898840 unknown :-1 s=CLOSED pgs=647 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/1099503874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819895bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 >> 172.21.15.90:0/1099503874 conn(0x5628197ed6a0 msgr2=0x562819888c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9bd7fff700 0 client.16707 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 shutdown_connections 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/1099503874 wait complete. 2024-04-06T14:13:16.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.921+0000 7f9c0ad44b80 20 client.16707 trim_cache size 0 max 16384 2024-04-06T14:13:16.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.943+0000 7f9c0ad44b80 1 Processor -- start 2024-04-06T14:13:16.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.943+0000 7f9c0ad44b80 1 -- start start 2024-04-06T14:13:16.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56281988e780 0x56281989fac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562819898de0 0x562819891420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562819897130 con 0x562819898de0 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281988b140 con 0x56281988e780 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5628198a0000 con 0x56281988f0a0 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:33954/0 (socket says 172.21.15.90:33954) 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 -- 172.21.15.90:0/885630837 learned_addr learned my addr 172.21.15.90:0/885630837 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56281988e780 0x56281989fac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bfd12d700 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562819898de0 0x562819891420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 -- 172.21.15.90:0/885630837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56281988e780 msgr2=0x56281989fac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56281988e780 0x56281989fac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 -- 172.21.15.90:0/885630837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562819898de0 msgr2=0x562819891420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562819898de0 0x562819891420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.944+0000 7f9bf7fff700 1 -- 172.21.15.90:0/885630837 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5628197a5ea0 con 0x56281988f0a0 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.945+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9bf0000f80 tx=0x7f9bf0013590 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cf87249d24d051be server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.945+0000 7f9bef7fe700 1 -- 172.21.15.90:0/885630837 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bf0031410 con 0x56281988f0a0 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.945+0000 7f9bef7fe700 1 -- 172.21.15.90:0/885630837 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bf0031570 con 0x56281988f0a0 2024-04-06T14:13:16.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.945+0000 7f9bef7fe700 1 -- 172.21.15.90:0/885630837 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bf0013af0 con 0x56281988f0a0 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/885630837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 msgr2=0x562819890ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f9bf0000f80 tx=0x7f9bf0013590 comp rx=0 tx=0).stop 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/885630837 shutdown_connections 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562819898de0 0x562819891420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56281988e780 0x56281989fac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/885630837 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x562819890ee0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/885630837 >> 172.21.15.90:0/885630837 conn(0x5628197ed6a0 msgr2=0x562819895460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/885630837 shutdown_connections 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/885630837 wait complete. 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 -- start start 2024-04-06T14:13:16.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.946+0000 7f9c0ad44b80 1 Processor -- start 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x56281989faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989dc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281989e390 con 0x56281988e780 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281989e500 con 0x562819898de0 2024-04-06T14:13:16.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56281989e6c0 con 0x56281988f0a0 2024-04-06T14:13:16.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9c0ad44b80 10 client.0 _ll_register_callbacks cb 0x562819785470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:16.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41166/0 (socket says 172.21.15.90:41166) 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfd12d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989dc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3062378933 learned_addr learned my addr 172.21.15.90:0/3062378933 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x56281989faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 msgr2=0x56281989faa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x56281989faa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3062378933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 msgr2=0x56281989dc80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989dc80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.947+0000 7f9bfc92c700 1 -- 172.21.15.90:0/3062378933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5628196613b0 con 0x56281988e780 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.948+0000 7f9bfc92c700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f9be8099890 tx=0x7f9be8004860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=546bb9a98726c4b0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.948+0000 7f9bd7fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.948+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9be80a18f0 con 0x56281988e780 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f9be8084170 con 0x56281988e780 2024-04-06T14:13:16.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9bf000f040 con 0x56281988e780 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9be80a1d80 con 0x56281988e780 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 10 client.16713 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562819665cd0 con 0x56281988e780 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bf000f040 con 0x56281988e780 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9be8066020 con 0x56281988e780 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 10 client.16713 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 10 client.16713 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 20 client.16713 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56281990e9b0 0x562819910dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 4 client.16713 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.949+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x56281962ecf0 con 0x56281990e9b0 2024-04-06T14:13:16.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56281990e9b0 0x562819910dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:16.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9bf7fff700 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56281990e9b0 0x562819910dd0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=a62c2f4fa4fe131f server_cookie=6e7b394c7df26f2f in_seq=0 out_seq=0 2024-04-06T14:13:16.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9bd7fff700 10 client.16713 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9bd7fff700 1 -- 172.21.15.90:0/3062378933 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x56281962ecf0 con 0x56281990e9b0 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9bd7fff700 10 client.16713 handle_command_reply: tid=0 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.950+0000 7f9c0ad44b80 1 client.16713 shutdown 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 msgr2=0x56281989f560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 secure :-1 s=READY pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f9be8099890 tx=0x7f9be8004860 comp rx=0 tx=0).stop 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 shutdown_connections 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56281990e9b0 0x562819910dd0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562819898de0 0x56281989dc80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56281988e780 0x56281989f560 secure :-1 s=CLOSED pgs=648 cs=0 l=1 rev1=1 crypto rx=0x7f9be8099890 tx=0x7f9be8004860 comp rx=0 tx=0).stop 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 --2- 172.21.15.90:0/3062378933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56281988f0a0 0x56281989faa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 >> 172.21.15.90:0/3062378933 conn(0x5628197ed6a0 msgr2=0x56281988d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9bd7fff700 0 client.16713 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:16.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 shutdown_connections 2024-04-06T14:13:16.957 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.951+0000 7f9c0ad44b80 1 -- 172.21.15.90:0/3062378933 wait complete. 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9c0ad44b80 20 client.16713 trim_cache size 0 max 16384 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 msgr2=0x7f9be40a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 0x7f9be40a3790 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f9bf81a7640 tx=0x7f9bf000b040 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 msgr2=0x7f9bf8112ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 0x7f9bf8112ba0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f9be800b450 tx=0x7f9be800b760 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 shutdown_connections 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9be40a12d0 0x7f9be40a3790 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9bf8113f00 0x7f9bf81aa970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9bf8107db0 0x7f9bf8112ba0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 --2- 172.21.15.90:0/3015003760 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9bf81086d0 0x7f9bf81130e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:16.958 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.952+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 >> 172.21.15.90:0/3015003760 conn(0x7f9bf806be00 msgr2=0x7f9bf806f9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:16.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.955+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 shutdown_connections 2024-04-06T14:13:16.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:16.955+0000 7f9bfeb90700 1 -- 172.21.15.90:0/3015003760 wait complete. 2024-04-06T14:13:16.971 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:16 smithi090 ceph-mon[43420]: pgmap v242: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 13 MiB/s wr, 683 op/s 2024-04-06T14:13:16.972 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:16 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3734167245' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:16.974 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:16.974 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:13:17.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:16 smithi134 ceph-mon[47549]: pgmap v242: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 13 MiB/s wr, 683 op/s 2024-04-06T14:13:17.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:16 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3734167245' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 -- 172.21.15.90:0/1448318366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 msgr2=0x7ff1d8103560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 --2- 172.21.15.90:0/1448318366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 0x7ff1d8103560 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7ff1c801a9a0 tx=0x7ff1c801acb0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 -- 172.21.15.90:0/1448318366 shutdown_connections 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 --2- 172.21.15.90:0/1448318366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 --2- 172.21.15.90:0/1448318366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 0x7ff1d8103560 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 --2- 172.21.15.90:0/1448318366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d8103b30 0x7ff1d8107b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.177+0000 7ff1dd347700 1 -- 172.21.15.90:0/1448318366 >> 172.21.15.90:0/1448318366 conn(0x7ff1d80fe9c0 msgr2=0x7ff1d8100de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.178+0000 7ff1dd347700 1 -- 172.21.15.90:0/1448318366 shutdown_connections 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.178+0000 7ff1dd347700 1 -- 172.21.15.90:0/1448318366 wait complete. 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.179+0000 7ff1dd347700 1 Processor -- start 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 -- start start 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 0x7ff1d810f8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 0x7ff1d810bff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d810c7c0 con 0x7ff1d810bb40 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d81bf650 con 0x7ff1d81082a0 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1dd347700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1d81bf790 con 0x7ff1d8103180 2024-04-06T14:13:17.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1d67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1d67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42974/0 (socket says 172.21.15.90:42974) 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.180+0000 7ff1d67fc700 1 -- 172.21.15.90:0/2155943769 learned_addr learned my addr 172.21.15.90:0/2155943769 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.181+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 0x7ff1d810bff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 -- 172.21.15.90:0/2155943769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 msgr2=0x7ff1d810f8d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 0x7ff1d810f8d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 -- 172.21.15.90:0/2155943769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 msgr2=0x7ff1d810fe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 -- 172.21.15.90:0/2155943769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1d0009c40 con 0x7ff1d810bb40 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.182+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 0x7ff1d810bff0 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7ff1d000efb0 tx=0x7ff1d000c450 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.183+0000 7ff1bffff700 1 -- 172.21.15.90:0/2155943769 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d000cb00 con 0x7ff1d810bb40 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.183+0000 7ff1bffff700 1 -- 172.21.15.90:0/2155943769 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1d000f040 con 0x7ff1d810bb40 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.183+0000 7ff1bffff700 1 -- 172.21.15.90:0/2155943769 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d0004660 con 0x7ff1d810bb40 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.183+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff1c801a600 con 0x7ff1d810bb40 2024-04-06T14:13:17.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.183+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1d8195980 con 0x7ff1d810bb40 2024-04-06T14:13:17.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.186+0000 7ff1bffff700 1 -- 172.21.15.90:0/2155943769 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff1d0007500 con 0x7ff1d810bb40 2024-04-06T14:13:17.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.187+0000 7ff1bffff700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 0x7ff1c00a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.187+0000 7ff1bffff700 1 -- 172.21.15.90:0/2155943769 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7ff1d00cc980 con 0x7ff1d810bb40 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.188+0000 7ff1e94fbb80 1 Processor -- start 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 -- start start 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af67370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7c450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6aeb6990 con 0x55ad6af7a060 2024-04-06T14:13:17.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6aeb4350 con 0x55ad6af678b0 2024-04-06T14:13:17.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.191+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6aeafc40 con 0x55ad6af66f90 2024-04-06T14:13:17.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.198+0000 7ff1d6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af67370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.198+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 0x7ff1c00a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.199+0000 7ff1d67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.199+0000 7ff1d67fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:42984/0 (socket says 172.21.15.90:42984) 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.199+0000 7ff1d67fc700 1 -- 172.21.15.90:0/197805625 learned_addr learned my addr 172.21.15.90:0/197805625 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7c450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d67fc700 1 -- 172.21.15.90:0/197805625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 msgr2=0x55ad6af67370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:17.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d67fc700 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af67370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d67fc700 1 -- 172.21.15.90:0/197805625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 msgr2=0x55ad6af7c450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:17.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d67fc700 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7c450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.200+0000 7ff1d67fc700 1 -- 172.21.15.90:0/197805625 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad6ad313b0 con 0x55ad6af678b0 2024-04-06T14:13:17.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.204+0000 7ff1d67fc700 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7ff1cc04da80 tx=0x7ff1cc04dd90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c54f40df971e79fc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.210+0000 7ff1bd7fa700 1 -- 172.21.15.90:0/197805625 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1cc051860 con 0x55ad6af678b0 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.210+0000 7ff1bd7fa700 1 -- 172.21.15.90:0/197805625 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1cc051e80 con 0x55ad6af678b0 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.210+0000 7ff1bd7fa700 1 -- 172.21.15.90:0/197805625 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1cc0563f0 con 0x55ad6af678b0 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/197805625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 msgr2=0x55ad6af637c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7ff1cc04da80 tx=0x7ff1cc04dd90 comp rx=0 tx=0).stop 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/197805625 shutdown_connections 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7c450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af67370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/197805625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af637c0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.211+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/197805625 >> 172.21.15.90:0/197805625 conn(0x55ad6aebd6a0 msgr2=0x55ad6af59a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.231+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 0x7ff1c00a37d0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7ff1c8006010 tx=0x7ff1c801a320 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.231+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/197805625 shutdown_connections 2024-04-06T14:13:17.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/197805625 wait complete. 2024-04-06T14:13:17.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 -- start start 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 Processor -- start 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af63640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af60930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6ae4bd80 con 0x55ad6af7a060 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6ae4ba20 con 0x55ad6af678b0 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6ad336a0 con 0x55ad6af66f90 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.234+0000 7ff1e94fbb80 10 client.0 _ll_register_callbacks cb 0x55ad6ae55470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41184/0 (socket says 172.21.15.90:41184) 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3599257944 learned_addr learned my addr 172.21.15.90:0/3599257944 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3599257944 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 msgr2=0x55ad6af63640 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:17.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af63640 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 msgr2=0x55ad6af60930 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:17.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af60930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.235+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3599257944 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1cc04d6b0 con 0x55ad6af7a060 2024-04-06T14:13:17.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.236+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00ca680 tx=0x7ff1d0099b80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=219dc307f267dc72 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.237+0000 7ff1baff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:17.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:16 smithi105 ceph-mon[46969]: pgmap v242: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 13 MiB/s wr, 683 op/s 2024-04-06T14:13:17.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:16 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3734167245' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:17.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.246+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d000ce60 con 0x55ad6af7a060 2024-04-06T14:13:17.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.246+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7ff1d0099d30 con 0x55ad6af7a060 2024-04-06T14:13:17.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.249+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ad6aea8700 con 0x55ad6af7a060 2024-04-06T14:13:17.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.252+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff1d0099ef0 con 0x55ad6af7a060 2024-04-06T14:13:17.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.252+0000 7ff1e94fbb80 10 client.16722 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:17.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.252+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad6acddb60 con 0x55ad6af7a060 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ad6aea8700 con 0x55ad6af7a060 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff1d00990e0 con 0x55ad6af7a060 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 10 client.16722 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 10 client.16722 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 20 client.16722 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6af6bf70 0x55ad6af6e390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 4 client.16722 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:17.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.253+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ad6acfecf0 con 0x55ad6af6bf70 2024-04-06T14:13:17.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.255+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6af6bf70 0x55ad6af6e390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.255+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6af6bf70 0x55ad6af6e390 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a620899aabd552c server_cookie=3baa7b3afc57908e in_seq=0 out_seq=0 2024-04-06T14:13:17.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.255+0000 7ff1baff5700 10 client.16722 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.256+0000 7ff1baff5700 1 -- 172.21.15.90:0/3599257944 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ad6acfecf0 con 0x55ad6af6bf70 2024-04-06T14:13:17.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.256+0000 7ff1baff5700 10 client.16722 handle_command_reply: tid=0 2024-04-06T14:13:17.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.257+0000 7ff1e94fbb80 1 client.16722 shutdown 2024-04-06T14:13:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 msgr2=0x55ad6af60ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 secure :-1 s=READY pgs=650 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00ca680 tx=0x7ff1d0099b80 comp rx=0 tx=0).stop 2024-04-06T14:13:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 shutdown_connections 2024-04-06T14:13:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af60ef0 unknown :-1 s=CLOSED pgs=650 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6af6bf70 0x55ad6af6e390 crc :-1 s=READY pgs=147 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af63640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3599257944 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af60930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.260+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 >> 172.21.15.90:0/3599257944 conn(0x55ad6aebd6a0 msgr2=0x55ad6af58c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.265+0000 7ff1baff5700 0 client.16722 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:17.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.265+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 shutdown_connections 2024-04-06T14:13:17.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.265+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3599257944 wait complete. 2024-04-06T14:13:17.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.266+0000 7ff1e94fbb80 20 client.16722 trim_cache size 0 max 16384 2024-04-06T14:13:17.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 Processor -- start 2024-04-06T14:13:17.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 -- start start 2024-04-06T14:13:17.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af60e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af85950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af593a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af86550 con 0x55ad6af7a060 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af5b220 con 0x55ad6af678b0 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.286+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af61a70 con 0x55ad6af66f90 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af60e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af60e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34004/0 (socket says 172.21.15.90:34004) 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d6ffd700 1 -- 172.21.15.90:0/3042407893 learned_addr learned my addr 172.21.15.90:0/3042407893 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:17.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af593a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3042407893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 msgr2=0x55ad6af60e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af60e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3042407893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 msgr2=0x55ad6af85950 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af85950 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.287+0000 7ff1d77fe700 1 -- 172.21.15.90:0/3042407893 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad6ae75ea0 con 0x55ad6af7a060 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.288+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af593a0 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7ff1d0096540 tx=0x7ff1d00a2040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ecf74d2f0c21c06 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.288+0000 7ff1bdffb700 1 -- 172.21.15.90:0/3042407893 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d00b9090 con 0x55ad6af7a060 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.288+0000 7ff1bdffb700 1 -- 172.21.15.90:0/3042407893 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1d00b91f0 con 0x55ad6af7a060 2024-04-06T14:13:17.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.288+0000 7ff1bdffb700 1 -- 172.21.15.90:0/3042407893 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d0099d30 con 0x55ad6af7a060 2024-04-06T14:13:17.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.289+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3042407893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 msgr2=0x55ad6af593a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.289+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af593a0 secure :-1 s=READY pgs=651 cs=0 l=1 rev1=1 crypto rx=0x7ff1d0096540 tx=0x7ff1d00a2040 comp rx=0 tx=0).stop 2024-04-06T14:13:17.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3042407893 shutdown_connections 2024-04-06T14:13:17.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af593a0 unknown :-1 s=CLOSED pgs=651 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af66f90 0x55ad6af60e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/3042407893 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af678b0 0x55ad6af85950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3042407893 >> 172.21.15.90:0/3042407893 conn(0x55ad6aebd6a0 msgr2=0x55ad6af85240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.296+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3042407893 shutdown_connections 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/3042407893 wait complete. 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 -- start start 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 Processor -- start 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af66f90 0x55ad6af80980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af678b0 0x55ad6af7ef00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af7f980 con 0x55ad6af7a060 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af7faf0 con 0x55ad6af66f90 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.298+0000 7ff1e94fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad6af7fcb0 con 0x55ad6af678b0 2024-04-06T14:13:17.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.299+0000 7ff1e94fbb80 10 client.0 _ll_register_callbacks cb 0x55ad6ae55470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:17.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.299+0000 7ff1d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.299+0000 7ff1d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41200/0 (socket says 172.21.15.90:41200) 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.299+0000 7ff1d77fe700 1 -- 172.21.15.90:0/803478688 learned_addr learned my addr 172.21.15.90:0/803478688 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 -- 172.21.15.90:0/803478688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af678b0 msgr2=0x55ad6af7ef00 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af678b0 0x55ad6af7ef00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 -- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af66f90 msgr2=0x55ad6af80980 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af66f90 0x55ad6af80980 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 -- 172.21.15.90:0/803478688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ad6acddb60 con 0x55ad6af7a060 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1d77fe700 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00ca130 tx=0x7ff1d009a310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0c4e41959a6c4fa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1baff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1d000ce60 con 0x55ad6af7a060 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(87..87 src has 1..87) v4 ==== 8600+0+0 (secure 0 0 0) 0x7ff1d0099d30 con 0x55ad6af7a060 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.300+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ad6ae70380 con 0x55ad6af7a060 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.301+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff1d00b2410 con 0x55ad6af7a060 2024-04-06T14:13:17.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.301+0000 7ff1e94fbb80 10 client.16728 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:17.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.301+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad6ad35cd0 con 0x55ad6af7a060 2024-04-06T14:13:17.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.301+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1d00ad080 con 0x55ad6af7a060 2024-04-06T14:13:17.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.301+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff1d00959d0 con 0x55ad6af7a060 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 10 client.16728 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 10 client.16728 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 20 client.16728 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6afefdf0 0x55ad6aff21e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 4 client.16728 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.302+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ad6ad33ce0 con 0x55ad6afefdf0 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6afefdf0 0x55ad6aff21e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1d6ffd700 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6afefdf0 0x55ad6aff21e0 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e5cbdc823ec8016d server_cookie=63484094abf05682 in_seq=0 out_seq=0 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1baff5700 10 client.16728 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1baff5700 1 -- 172.21.15.90:0/803478688 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ad6ad33ce0 con 0x55ad6afefdf0 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1baff5700 10 client.16728 handle_command_reply: tid=0 2024-04-06T14:13:17.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.303+0000 7ff1e94fbb80 1 client.16728 shutdown 2024-04-06T14:13:17.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 msgr2=0x55ad6af7f440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 secure :-1 s=READY pgs=652 cs=0 l=1 rev1=1 crypto rx=0x7ff1d00ca130 tx=0x7ff1d009a310 comp rx=0 tx=0).stop 2024-04-06T14:13:17.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 shutdown_connections 2024-04-06T14:13:17.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad6af7a060 0x55ad6af7f440 unknown :-1 s=CLOSED pgs=652 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad6af66f90 0x55ad6af80980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad6afefdf0 0x55ad6aff21e0 crc :-1 s=READY pgs=148 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 --2- 172.21.15.90:0/803478688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad6af678b0 0x55ad6af7ef00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 >> 172.21.15.90:0/803478688 conn(0x55ad6aebd6a0 msgr2=0x55ad6af65e30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1baff5700 0 client.16728 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 shutdown_connections 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.304+0000 7ff1e94fbb80 1 -- 172.21.15.90:0/803478688 wait complete. 2024-04-06T14:13:17.306 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1e94fbb80 20 client.16728 trim_cache size 0 max 16384 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 msgr2=0x7ff1c00a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 0x7ff1c00a37d0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7ff1c8006010 tx=0x7ff1c801a320 comp rx=0 tx=0).stop 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 msgr2=0x7ff1d810bff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 0x7ff1d810bff0 secure :-1 s=READY pgs=649 cs=0 l=1 rev1=1 crypto rx=0x7ff1d000efb0 tx=0x7ff1d000c450 comp rx=0 tx=0).stop 2024-04-06T14:13:17.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 shutdown_connections 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1c00a1310 0x7ff1c00a37d0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1d810bb40 0x7ff1d810bff0 unknown :-1 s=CLOSED pgs=649 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1d8103180 0x7ff1d810f8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 --2- 172.21.15.90:0/2155943769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1d81082a0 0x7ff1d810fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.305+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 >> 172.21.15.90:0/2155943769 conn(0x7ff1d80fe9c0 msgr2=0x7ff1d80ff550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:17.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.306+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 shutdown_connections 2024-04-06T14:13:17.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:17.309+0000 7ff1dd347700 1 -- 172.21.15.90:0/2155943769 wait complete. 2024-04-06T14:13:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 450.0 s, 1090.2 tps, lat 91.400 ms stddev 119.242 2024-04-06T14:13:18.330 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:13:18.330 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:13:18.330 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2406729674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 msgr2=0x7f8e580ff9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2406729674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e580ff9c0 secure :-1 s=READY pgs=653 cs=0 l=1 rev1=1 crypto rx=0x7f8e54009990 tx=0x7f8e54009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2406729674 shutdown_connections 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2406729674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8e58104700 0x7f8e5810c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2406729674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e580ff9c0 unknown :-1 s=CLOSED pgs=653 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2406729674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e580fff90 0x7f8e58103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2406729674 >> 172.21.15.90:0/2406729674 conn(0x7f8e580761b0 msgr2=0x7f8e580fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2406729674 shutdown_connections 2024-04-06T14:13:18.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.532+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2406729674 wait complete. 2024-04-06T14:13:18.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.533+0000 7f8e5fe13700 1 Processor -- start 2024-04-06T14:13:18.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.534+0000 7f8e5fe13700 1 -- start start 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8e58104700 0x7f8e581976d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e5819ba70 0x7f8e5819bef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5819c4f0 con 0x7f8e580ff5e0 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5819c660 con 0x7f8e5819ba70 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5fe13700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e5819c820 con 0x7f8e58104700 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41228/0 (socket says 172.21.15.90:41228) 2024-04-06T14:13:18.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 -- 172.21.15.90:0/2583560412 learned_addr learned my addr 172.21.15.90:0/2583560412 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e5819ba70 0x7f8e5819bef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 -- 172.21.15.90:0/2583560412 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8e58104700 msgr2=0x7f8e581976d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8e58104700 0x7f8e581976d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 -- 172.21.15.90:0/2583560412 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e5819ba70 msgr2=0x7f8e5819bef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e5819ba70 0x7f8e5819bef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.535+0000 7f8e5dbaf700 1 -- 172.21.15.90:0/2583560412 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e540095f0 con 0x7f8e580ff5e0 2024-04-06T14:13:18.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f8e54004760 tx=0x7f8e54004790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e4effd700 1 -- 172.21.15.90:0/2583560412 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e5401d070 con 0x7f8e580ff5e0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e4effd700 1 -- 172.21.15.90:0/2583560412 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e54022470 con 0x7f8e580ff5e0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e581a3d20 con 0x7f8e580ff5e0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e4effd700 1 -- 172.21.15.90:0/2583560412 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e54017650 con 0x7f8e580ff5e0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.536+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e581a4140 con 0x7f8e580ff5e0 2024-04-06T14:13:18.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.538+0000 7f8e4effd700 1 -- 172.21.15.90:0/2583560412 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8e540177b0 con 0x7f8e580ff5e0 2024-04-06T14:13:18.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.539+0000 7f8e4effd700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 0x7f8e440a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.539+0000 7f8e4effd700 1 -- 172.21.15.90:0/2583560412 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8e540cdf60 con 0x7f8e580ff5e0 2024-04-06T14:13:18.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.539+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 0x7f8e440a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.540+0000 7f8e6bfc7b80 1 Processor -- start 2024-04-06T14:13:18.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 -- start start 2024-04-06T14:13:18.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a54be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a5e920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b819aca10 con 0x564b81a5ee60 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b819aa3d0 con 0x564b81a54800 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b819a5cc0 con 0x564b81a55120 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e5e3b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e5e3b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41232/0 (socket says 172.21.15.90:41232) 2024-04-06T14:13:18.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.541+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/3091582752 learned_addr learned my addr 172.21.15.90:0/3091582752 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a54be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 0x7f8e440a38a0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8e581986e0 tx=0x7f8e48009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a5e920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/3091582752 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 msgr2=0x564b81a5e920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a5e920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/3091582752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 msgr2=0x564b81a54be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a54be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/3091582752 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564b81827430 con 0x564b81a5ee60 2024-04-06T14:13:18.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.543+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f8e50011680 tx=0x7f8e50009920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=206d9636f38bf3d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.544+0000 7f8e43fff700 1 -- 172.21.15.90:0/3091582752 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e5000e040 con 0x564b81a5ee60 2024-04-06T14:13:18.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.544+0000 7f8e43fff700 1 -- 172.21.15.90:0/3091582752 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e5000f040 con 0x564b81a5ee60 2024-04-06T14:13:18.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.544+0000 7f8e43fff700 1 -- 172.21.15.90:0/3091582752 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e50004280 con 0x564b81a5ee60 2024-04-06T14:13:18.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.545+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/3091582752 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 msgr2=0x564b81a61250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.545+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 secure :-1 s=READY pgs=655 cs=0 l=1 rev1=1 crypto rx=0x7f8e50011680 tx=0x7f8e50009920 comp rx=0 tx=0).stop 2024-04-06T14:13:18.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/3091582752 shutdown_connections 2024-04-06T14:13:18.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a61250 unknown :-1 s=CLOSED pgs=655 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a54be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/3091582752 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a5e920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/3091582752 >> 172.21.15.90:0/3091582752 conn(0x564b819b3720 msgr2=0x564b81a4fb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.546+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/3091582752 shutdown_connections 2024-04-06T14:13:18.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.547+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/3091582752 wait complete. 2024-04-06T14:13:18.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.547+0000 7f8e6bfc7b80 1 -- start start 2024-04-06T14:13:18.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.547+0000 7f8e6bfc7b80 1 Processor -- start 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a5e900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81a5c380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81941e00 con 0x564b81a54800 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81941aa0 con 0x564b81a55120 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81829720 con 0x564b81a5ee60 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e6bfc7b80 10 client.0 _ll_register_callbacks cb 0x564b8194b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43026/0 (socket says 172.21.15.90:43026) 2024-04-06T14:13:18.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/53660228 learned_addr learned my addr 172.21.15.90:0/53660228 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81a5c380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a5e900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/53660228 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 msgr2=0x564b81a5c380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81a5c380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/53660228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 msgr2=0x564b81a5e900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a5e900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/53660228 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e50011330 con 0x564b81a55120 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.548+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a5e900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f8e480104b0 tx=0x7f8e480047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6df9c62d9a9e4ab4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e48026470 con 0x564b81a55120 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564b81966400 con 0x564b81a55120 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8e480265d0 con 0x564b81a55120 2024-04-06T14:13:18.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8e48026790 con 0x564b81a55120 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e6bfc7b80 10 client.26462 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564b817d3be0 con 0x564b81a55120 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564b81966400 con 0x564b81a55120 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.549+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8e48018030 con 0x564b81a55120 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 10 client.26462 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 10 client.26462 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 20 client.26462 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81a6ea40 0x564b81a70e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 4 client.26462 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:18.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564b8199e780 con 0x564b81a6ea40 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81a6ea40 0x564b81a70e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e417fa700 10 client.26462 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.550+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81a6ea40 0x564b81a70e60 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=484dd85e3c39f5ef server_cookie=4a99dbcc4bf44016 in_seq=0 out_seq=0 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e417fa700 1 -- 172.21.15.90:0/53660228 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564b8199e780 con 0x564b81a6ea40 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e417fa700 10 client.26462 handle_command_reply: tid=0 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 client.26462 shutdown 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 msgr2=0x564b81a5bc40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f8e480104b0 tx=0x7f8e480047c0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 shutdown_connections 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81a6ea40 0x564b81a70e60 crc :-1 s=READY pgs=149 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81a5c380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a5e900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e417fa700 0 client.26462 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/53660228 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a5bc40 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 >> 172.21.15.90:0/53660228 conn(0x564b819b3720 msgr2=0x564b81a4eca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 shutdown_connections 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.551+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/53660228 wait complete. 2024-04-06T14:13:18.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.552+0000 7f8e6bfc7b80 20 client.26462 trim_cache size 0 max 16384 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 Processor -- start 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 -- start start 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a4fcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a65b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81a55a90 con 0x564b81a5ee60 2024-04-06T14:13:18.578 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81a5bfc0 con 0x564b81a54800 2024-04-06T14:13:18.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.576+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81a6c1a0 con 0x564b81a55120 2024-04-06T14:13:18.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41260/0 (socket says 172.21.15.90:41260) 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/2931170569 learned_addr learned my addr 172.21.15.90:0/2931170569 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/2931170569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 msgr2=0x564b81a65b10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a65b10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/2931170569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 msgr2=0x564b81a4fcb0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a4fcb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 -- 172.21.15.90:0/2931170569 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564b81a75a90 con 0x564b81a5ee60 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.577+0000 7f8e5e3b0700 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x564b81a56270 tx=0x7f8e500060b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=acf83df30777ffdb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.578+0000 7f8e4cff9700 1 -- 172.21.15.90:0/2931170569 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e5000e040 con 0x564b81a5ee60 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.578+0000 7f8e4cff9700 1 -- 172.21.15.90:0/2931170569 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e50003c00 con 0x564b81a5ee60 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.578+0000 7f8e4cff9700 1 -- 172.21.15.90:0/2931170569 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e50007760 con 0x564b81a5ee60 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.578+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2931170569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 msgr2=0x564b81a5e430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.578+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 secure :-1 s=READY pgs=656 cs=0 l=1 rev1=1 crypto rx=0x564b81a56270 tx=0x7f8e500060b0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.579+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2931170569 shutdown_connections 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.579+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a5ee60 0x564b81a5e430 unknown :-1 s=CLOSED pgs=656 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.579+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a54800 0x564b81a4fcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.579+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2931170569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a55120 0x564b81a65b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.579+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2931170569 >> 172.21.15.90:0/2931170569 conn(0x564b819b3720 msgr2=0x564b81a4f5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.581+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2931170569 shutdown_connections 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.581+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2931170569 wait complete. 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.581+0000 7f8e6bfc7b80 1 -- start start 2024-04-06T14:13:18.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.581+0000 7f8e6bfc7b80 1 Processor -- start 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81abbe60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81abc410 con 0x564b81a54800 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81abc580 con 0x564b81a55120 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b81abc740 con 0x564b81a5ee60 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e6bfc7b80 10 client.0 _ll_register_callbacks cb 0x564b8194b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5dbaf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5e3b0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81abbe60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43032/0 (socket says 172.21.15.90:43032) 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/2427902088 learned_addr learned my addr 172.21.15.90:0/2427902088 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5dbaf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41274/0 (socket says 172.21.15.90:41274) 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/2427902088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 msgr2=0x564b81abbe60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81abbe60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/2427902088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 msgr2=0x564b81a650d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5d3ae700 1 -- 172.21.15.90:0/2427902088 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564b81827430 con 0x564b81a55120 2024-04-06T14:13:18.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.584+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.586+0000 7f8e5d3ae700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f8e4801da50 tx=0x7f8e4800e820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5b1c3c864923b66f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.586+0000 7f8e417fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.586+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e4801d640 con 0x564b81a55120 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.586+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8e4801d7a0 con 0x564b81a55120 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.586+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8e5000f040 con 0x564b81a55120 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.587+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8e48004830 con 0x564b81a55120 2024-04-06T14:13:18.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.587+0000 7f8e6bfc7b80 10 client.26465 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:18.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.587+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564b8196bf20 con 0x564b81a55120 2024-04-06T14:13:18.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.587+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e5000f040 con 0x564b81a55120 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8e48035070 con 0x564b81a55120 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 10 client.26465 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 10 client.26465 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 20 client.26465 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81ac51f0 0x564b81ac7610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 4 client.26465 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:13:18.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564b817f4d70 con 0x564b81ac51f0 2024-04-06T14:13:18.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.589+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81ac51f0 0x564b81ac7610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.590+0000 7f8e5dbaf700 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81ac51f0 0x564b81ac7610 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e927b189b5566f8a server_cookie=d06e9734d75b3a9c in_seq=0 out_seq=0 2024-04-06T14:13:18.590 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.590+0000 7f8e417fa700 10 client.26465 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e417fa700 1 -- 172.21.15.90:0/2427902088 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564b817f4d70 con 0x564b81ac51f0 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e417fa700 10 client.26465 handle_command_reply: tid=0 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 client.26465 shutdown 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 msgr2=0x564b81a65610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f8e4801da50 tx=0x7f8e4800e820 comp rx=0 tx=0).stop 2024-04-06T14:13:18.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 shutdown_connections 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b81ac51f0 0x564b81ac7610 crc :-1 s=READY pgs=150 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b81a5ee60 0x564b81abbe60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b81a54800 0x564b81a650d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 --2- 172.21.15.90:0/2427902088 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b81a55120 0x564b81a65610 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e417fa700 0 client.26465 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 >> 172.21.15.90:0/2427902088 conn(0x564b819b3720 msgr2=0x564b81a5a510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.591+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 shutdown_connections 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e6bfc7b80 1 -- 172.21.15.90:0/2427902088 wait complete. 2024-04-06T14:13:18.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e6bfc7b80 20 client.26465 trim_cache size 0 max 16384 2024-04-06T14:13:18.594 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:18.594 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:13:18.594 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "082aed95-6811-479e-9e5e-a956be9030ca", 2024-04-06T14:13:18.594 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:13:18.594 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 msgr2=0x7f8e440a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 0x7f8e440a38a0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f8e581986e0 tx=0x7f8e48009260 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 msgr2=0x7f8e58197190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 secure :-1 s=READY pgs=654 cs=0 l=1 rev1=1 crypto rx=0x7f8e54004760 tx=0x7f8e54004790 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 shutdown_connections 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8e440a13e0 0x7f8e440a38a0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e5819ba70 0x7f8e5819bef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8e580ff5e0 0x7f8e58197190 unknown :-1 s=CLOSED pgs=654 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 --2- 172.21.15.90:0/2583560412 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8e58104700 0x7f8e581976d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.592+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 >> 172.21.15.90:0/2583560412 conn(0x7f8e580761b0 msgr2=0x7f8e5810fc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.593+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 shutdown_connections 2024-04-06T14:13:18.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.593+0000 7f8e5fe13700 1 -- 172.21.15.90:0/2583560412 wait complete. 2024-04-06T14:13:18.623 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:13:18.623 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 -- 172.21.15.90:0/702955511 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 msgr2=0x7fea9810dc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/702955511 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea9810dc10 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fea90009f90 tx=0x7fea9000b330 comp rx=0 tx=0).stop 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 -- 172.21.15.90:0/702955511 shutdown_connections 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/702955511 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea9810dc10 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/702955511 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/702955511 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea98101920 0x7fea98105960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 -- 172.21.15.90:0/702955511 >> 172.21.15.90:0/702955511 conn(0x7fea980fc7f0 msgr2=0x7fea980fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 -- 172.21.15.90:0/702955511 shutdown_connections 2024-04-06T14:13:18.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.866+0000 7fea9d4d9700 1 -- 172.21.15.90:0/702955511 wait complete. 2024-04-06T14:13:18.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.867+0000 7fea9d4d9700 1 Processor -- start 2024-04-06T14:13:18.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 -- start start 2024-04-06T14:13:18.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98070360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea980708c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 0x7fea9806b970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9806c080 con 0x7fea98100f70 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9806c1f0 con 0x7fea98106090 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea9d4d9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea980740d0 con 0x7fea9806b4c0 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98070360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98070360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41298/0 (socket says 172.21.15.90:41298) 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea96ffd700 1 -- 172.21.15.90:0/1488540423 learned_addr learned my addr 172.21.15.90:0/1488540423 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea977fe700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 0x7fea9806b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.868+0000 7fea967fc700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea980708c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 -- 172.21.15.90:0/1488540423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 msgr2=0x7fea980708c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea980708c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 -- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 msgr2=0x7fea98070360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98070360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 -- 172.21.15.90:0/1488540423 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea90009c40 con 0x7fea9806b4c0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea977fe700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 0x7fea9806b970 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fea9000b6c0 tx=0x7fea90008fb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea7ffff700 1 -- 172.21.15.90:0/1488540423 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea9000e040 con 0x7fea9806b4c0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea7ffff700 1 -- 172.21.15.90:0/1488540423 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea9001d470 con 0x7fea9806b4c0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea7ffff700 1 -- 172.21.15.90:0/1488540423 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea900052a0 con 0x7fea9806b4c0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea98074350 con 0x7fea9806b4c0 2024-04-06T14:13:18.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.869+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea98074810 con 0x7fea9806b4c0 2024-04-06T14:13:18.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.873+0000 7fea7ffff700 1 -- 172.21.15.90:0/1488540423 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fea9001d5e0 con 0x7fea9806b4c0 2024-04-06T14:13:18.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.875+0000 7fea7ffff700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 0x7fea800a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.875+0000 7fea7ffff700 1 -- 172.21.15.90:0/1488540423 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fea900ce080 con 0x7fea9806b4c0 2024-04-06T14:13:18.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.875+0000 7feaa968db80 1 Processor -- start 2024-04-06T14:13:18.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 0x7fea800a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 -- start start 2024-04-06T14:13:18.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb7698a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb769de0 0x55d2bb76c1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb6b7990 con 0x55d2bb769de0 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb6b5350 con 0x55d2bb7600a0 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7feaa968db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb6b0c40 con 0x55d2bb75f780 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.876+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 0x7fea800a3850 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fea8c005950 tx=0x7fea8c009310 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34068/0 (socket says 172.21.15.90:34068) 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 -- 172.21.15.90:0/1043579005 learned_addr learned my addr 172.21.15.90:0/1043579005 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea977fe700 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb769de0 0x55d2bb76c1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 -- 172.21.15.90:0/1043579005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 msgr2=0x55d2bb7698a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb7698a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 -- 172.21.15.90:0/1043579005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb769de0 msgr2=0x55d2bb76c1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb769de0 0x55d2bb76c1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 -- 172.21.15.90:0/1043579005 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d2bb5323b0 con 0x55d2bb75f780 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fea8c004210 tx=0x7fea8c014000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4bbfbc4202c38560 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea7d7fa700 1 -- 172.21.15.90:0/1043579005 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8c003d10 con 0x55d2bb75f780 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea7d7fa700 1 -- 172.21.15.90:0/1043579005 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea8c003e70 con 0x55d2bb75f780 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.877+0000 7fea7d7fa700 1 -- 172.21.15.90:0/1043579005 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8c01d730 con 0x55d2bb75f780 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 -- 172.21.15.90:0/1043579005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 msgr2=0x55d2bb75fb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fea8c004210 tx=0x7fea8c014000 comp rx=0 tx=0).stop 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 -- 172.21.15.90:0/1043579005 shutdown_connections 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb769de0 0x55d2bb76c1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb75fb60 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 --2- 172.21.15.90:0/1043579005 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb7698a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 -- 172.21.15.90:0/1043579005 >> 172.21.15.90:0/1043579005 conn(0x55d2bb6be6a0 msgr2=0x55d2bb75aa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 -- 172.21.15.90:0/1043579005 shutdown_connections 2024-04-06T14:13:18.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.878+0000 7feaa968db80 1 -- 172.21.15.90:0/1043579005 wait complete. 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 -- start start 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 Processor -- start 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb766bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb767650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb64cd80 con 0x55d2bb75f780 2024-04-06T14:13:18.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb64ca20 con 0x55d2bb769de0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb5346a0 con 0x55d2bb7600a0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7feaa968db80 10 client.0 _ll_register_callbacks cb 0x55d2bb656470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34078/0 (socket says 172.21.15.90:34078) 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 -- 172.21.15.90:0/1713233577 learned_addr learned my addr 172.21.15.90:0/1713233577 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea977fe700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb767650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb766bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 -- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 msgr2=0x55d2bb767650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb767650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 -- 172.21.15.90:0/1713233577 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 msgr2=0x55d2bb766bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb766bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.879+0000 7fea967fc700 1 -- 172.21.15.90:0/1713233577 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea8c017070 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb766bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea967fc700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fea880116c0 tx=0x7fea880119d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a0250bed3fd26af7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea7aff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8800e680 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fea8800e7e0 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d2bb671380 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fea8800e9a0 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7feaa968db80 10 client.26308 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.880+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d2bb4deb60 con 0x55d2bb7600a0 2024-04-06T14:13:18.887 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.882+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea88026020 con 0x55d2bb7600a0 2024-04-06T14:13:18.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fea88021b40 con 0x55d2bb7600a0 2024-04-06T14:13:18.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 10 client.26308 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:18.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 10 client.26308 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:18.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 20 client.26308 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:18.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb779a50 0x55d2bb77be70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 4 client.26308 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d2bb6a9700 con 0x55d2bb779a50 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb779a50 0x55d2bb77be70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.883+0000 7fea96ffd700 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb779a50 0x55d2bb77be70 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d5dcc539cac11ca0 server_cookie=f34d3121547823e3 in_seq=0 out_seq=0 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7fea7aff5700 10 client.26308 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7fea7aff5700 1 -- 172.21.15.90:0/1713233577 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d2bb6a9700 con 0x55d2bb779a50 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7fea7aff5700 10 client.26308 handle_command_reply: tid=0 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 client.26308 shutdown 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 msgr2=0x55d2bb767110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fea880116c0 tx=0x7fea880119d0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 shutdown_connections 2024-04-06T14:13:18.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb767650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb766bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb779a50 0x55d2bb77be70 crc :-1 s=READY pgs=151 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 --2- 172.21.15.90:0/1713233577 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb7600a0 0x55d2bb767110 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7fea7aff5700 0 client.26308 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.886+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 >> 172.21.15.90:0/1713233577 conn(0x55d2bb6be6a0 msgr2=0x55d2bb759c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.887+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 shutdown_connections 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.887+0000 7feaa968db80 1 -- 172.21.15.90:0/1713233577 wait complete. 2024-04-06T14:13:18.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.887+0000 7feaa968db80 20 client.26308 trim_cache size 0 max 16384 2024-04-06T14:13:18.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 Processor -- start 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 -- start start 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb75a8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb769de0 0x55d2bb763e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb7681d0 con 0x55d2bb75f780 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb75bdc0 con 0x55d2bb7600a0 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7feaa968db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb75b700 con 0x55d2bb769de0 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:41326/0 (socket says 172.21.15.90:41326) 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 -- 172.21.15.90:0/2933376251 learned_addr learned my addr 172.21.15.90:0/2933376251 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea967fc700 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb75a8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea977fe700 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb769de0 0x55d2bb763e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 -- 172.21.15.90:0/2933376251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb769de0 msgr2=0x55d2bb763e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb769de0 0x55d2bb763e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 -- 172.21.15.90:0/2933376251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 msgr2=0x55d2bb75a8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb75a8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.917 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.913+0000 7fea96ffd700 1 -- 172.21.15.90:0/2933376251 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d2bb676ea0 con 0x55d2bb75f780 2024-04-06T14:13:18.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.914+0000 7fea96ffd700 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fea8c0093f0 tx=0x7fea8c009380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=86bf7e3cf0f60a65 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.914+0000 7fea7dffb700 1 -- 172.21.15.90:0/2933376251 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8c02a040 con 0x55d2bb75f780 2024-04-06T14:13:18.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.914+0000 7fea7dffb700 1 -- 172.21.15.90:0/2933376251 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea8c032bf0 con 0x55d2bb75f780 2024-04-06T14:13:18.918 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.914+0000 7fea7dffb700 1 -- 172.21.15.90:0/2933376251 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8c005550 con 0x55d2bb75f780 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.915+0000 7feaa968db80 1 -- 172.21.15.90:0/2933376251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 msgr2=0x55d2bb75a3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.915+0000 7feaa968db80 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 secure :-1 s=READY pgs=657 cs=0 l=1 rev1=1 crypto rx=0x7fea8c0093f0 tx=0x7fea8c009380 comp rx=0 tx=0).stop 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 -- 172.21.15.90:0/2933376251 shutdown_connections 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb769de0 0x55d2bb763e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb75f780 0x55d2bb75a3a0 unknown :-1 s=CLOSED pgs=657 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 --2- 172.21.15.90:0/2933376251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb7600a0 0x55d2bb75a8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 -- 172.21.15.90:0/2933376251 >> 172.21.15.90:0/2933376251 conn(0x55d2bb6be6a0 msgr2=0x55d2bb7654a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 -- 172.21.15.90:0/2933376251 shutdown_connections 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 -- 172.21.15.90:0/2933376251 wait complete. 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 -- start start 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.916+0000 7feaa968db80 1 Processor -- start 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 0x55d2bb76f7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb76fd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb7cd890 con 0x55d2bb7600a0 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb7cd9d0 con 0x55d2bb769de0 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d2bb7cdb10 con 0x55d2bb75f780 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7feaa968db80 10 client.0 _ll_register_callbacks cb 0x55d2bb656470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:34084/0 (socket says 172.21.15.90:34084) 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 -- 172.21.15.90:0/3597671365 learned_addr learned my addr 172.21.15.90:0/3597671365 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea967fc700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 0x55d2bb76f7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea977fe700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb76fd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 -- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 msgr2=0x55d2bb76fd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb76fd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 -- 172.21.15.90:0/3597671365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 msgr2=0x55d2bb76f7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 0x55d2bb76f7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 -- 172.21.15.90:0/3597671365 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d2bb5323b0 con 0x55d2bb75f780 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea967fc700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 0x55d2bb76f7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea96ffd700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fea8c00a010 tx=0x7fea8c011b30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b821aa4773e0993b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.917+0000 7fea7aff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8c02a050 con 0x55d2bb75f780 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fea8c032310 con 0x55d2bb75f780 2024-04-06T14:13:18.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fea8c003e70 con 0x55d2bb75f780 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fea8c0324d0 con 0x55d2bb75f780 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 10 client.26314 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d2bb536cd0 con 0x55d2bb75f780 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea8c003e70 con 0x55d2bb75f780 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fea8c024020 con 0x55d2bb75f780 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 10 client.26314 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 10 client.26314 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 20 client.26314 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb7cede0 0x55d2bb7d1200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 4 client.26314 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.918+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d2bb4ffcf0 con 0x55d2bb7cede0 2024-04-06T14:13:18.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7fea967fc700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb7cede0 0x55d2bb7d1200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7fea7aff5700 10 client.26314 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7fea967fc700 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb7cede0 0x55d2bb7d1200 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2f6f848e4711f0a6 server_cookie=d42ce3cf4d90729c in_seq=0 out_seq=0 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7fea7aff5700 1 -- 172.21.15.90:0/3597671365 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x7fea88026110 con 0x55d2bb7cede0 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7fea7aff5700 10 client.26314 handle_command_reply: tid=0 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 client.26314 shutdown 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 msgr2=0x55d2bb76f2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fea8c00a010 tx=0x7fea8c011b30 comp rx=0 tx=0).stop 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 shutdown_connections 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d2bb769de0 0x55d2bb76fd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d2bb7cede0 0x55d2bb7d1200 crc :-1 s=READY pgs=152 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d2bb75f780 0x55d2bb76f2b0 secure :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fea8c00a010 tx=0x7fea8c011b30 comp rx=0 tx=0).stop 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 --2- 172.21.15.90:0/3597671365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d2bb7600a0 0x55d2bb76f7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.919+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 >> 172.21.15.90:0/3597671365 conn(0x55d2bb6be6a0 msgr2=0x55d2bb766b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.920+0000 7fea7aff5700 0 client.26314 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.920+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 shutdown_connections 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.920+0000 7feaa968db80 1 -- 172.21.15.90:0/3597671365 wait complete. 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.920+0000 7feaa968db80 20 client.26314 trim_cache size 0 max 16384 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "082aed95-6811-479e-9e5e-a956be9030ca": { 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "082aed95-6811-479e-9e5e-a956be9030ca", 2024-04-06T14:13:18.928 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:13:18.930 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:13:18.930 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:13:18.930 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 msgr2=0x7fea800a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 0x7fea800a3850 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fea8c005950 tx=0x7fea8c009310 comp rx=0 tx=0).stop 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 msgr2=0x7fea9806b970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 0x7fea9806b970 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fea9000b6c0 tx=0x7fea90008fb0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 shutdown_connections 2024-04-06T14:13:18.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fea800a1390 0x7fea800a3850 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fea9806b4c0 0x7fea9806b970 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fea98100f70 0x7fea98070360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 --2- 172.21.15.90:0/1488540423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fea98106090 0x7fea980708c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 >> 172.21.15.90:0/1488540423 conn(0x7fea980fc7f0 msgr2=0x7fea980feb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 shutdown_connections 2024-04-06T14:13:18.932 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:18.921+0000 7fea9d4d9700 1 -- 172.21.15.90:0/1488540423 wait complete. 2024-04-06T14:13:18.939 INFO:tasks.cephfs.filesystem:scrub status for tag:082aed95-6811-479e-9e5e-a956be9030ca - {'path': '/', 'tag': '082aed95-6811-479e-9e5e-a956be9030ca', 'options': 'recursive,force'} 2024-04-06T14:13:18.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:18 smithi105 ceph-mon[46969]: osdmap e88: 12 total, 12 up, 12 in 2024-04-06T14:13:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:18 smithi105 ceph-mon[46969]: pgmap v244: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 17 MiB/s wr, 847 op/s 2024-04-06T14:13:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:18 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:13:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:18 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:18.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:18 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:13:19.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:18 smithi090 ceph-mon[43420]: osdmap e88: 12 total, 12 up, 12 in 2024-04-06T14:13:19.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:18 smithi090 ceph-mon[43420]: pgmap v244: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 17 MiB/s wr, 847 op/s 2024-04-06T14:13:19.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:18 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:13:19.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:18 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:19.053 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:18 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:13:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:18 smithi134 ceph-mon[47549]: osdmap e88: 12 total, 12 up, 12 in 2024-04-06T14:13:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:18 smithi134 ceph-mon[47549]: pgmap v244: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 17 MiB/s wr, 847 op/s 2024-04-06T14:13:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:18 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:13:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:18 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:18 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:13:19.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:19 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:20.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:19 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:20.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:19 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:20.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:20 smithi105 ceph-mon[46969]: pgmap v245: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 9.0 MiB/s wr, 407 op/s 2024-04-06T14:13:20.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:20 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:13:21.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:20 smithi090 ceph-mon[43420]: pgmap v245: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 9.0 MiB/s wr, 407 op/s 2024-04-06T14:13:21.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:20 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:13:21.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:20 smithi134 ceph-mon[47549]: pgmap v245: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 9.0 MiB/s wr, 407 op/s 2024-04-06T14:13:21.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:20 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:13:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 455.0 s, 1669.2 tps, lat 60.382 ms stddev 69.124 2024-04-06T14:13:23.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:22 smithi105 ceph-mon[46969]: pgmap v246: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:23.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:22 smithi090 ceph-mon[43420]: pgmap v246: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:23.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:22 smithi134 ceph-mon[47549]: pgmap v246: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:24.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:23 smithi134 ceph-mon[47549]: pgmap v247: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:24.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:23 smithi105 ceph-mon[46969]: pgmap v247: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:24.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:23 smithi090 ceph-mon[43420]: pgmap v247: 97 pgs: 97 active+clean; 7.2 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 17 MiB/s wr, 766 op/s 2024-04-06T14:13:27.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:26 smithi134 ceph-mon[47549]: pgmap v248: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 885 op/s 2024-04-06T14:13:27.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:26 smithi105 ceph-mon[46969]: pgmap v248: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 885 op/s 2024-04-06T14:13:27.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:26 smithi090 ceph-mon[43420]: pgmap v248: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 885 op/s 2024-04-06T14:13:28.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 460.0 s, 1312.5 tps, lat 76.080 ms stddev 87.975 2024-04-06T14:13:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:28 smithi090 ceph-mon[43420]: pgmap v249: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 879 op/s 2024-04-06T14:13:29.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:28 smithi134 ceph-mon[47549]: pgmap v249: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 879 op/s 2024-04-06T14:13:29.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:28 smithi105 ceph-mon[46969]: pgmap v249: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 20 MiB/s wr, 879 op/s 2024-04-06T14:13:31.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:30 smithi105 ceph-mon[46969]: pgmap v250: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 737 op/s 2024-04-06T14:13:31.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:30 smithi090 ceph-mon[43420]: pgmap v250: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 737 op/s 2024-04-06T14:13:31.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:30 smithi134 ceph-mon[47549]: pgmap v250: 97 pgs: 97 active+clean; 7.3 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 16 MiB/s wr, 737 op/s 2024-04-06T14:13:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 465.0 s, 1342.5 tps, lat 75.174 ms stddev 143.816 2024-04-06T14:13:33.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:32 smithi134 ceph-mon[47549]: pgmap v251: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 34 MiB/s wr, 1.94k op/s 2024-04-06T14:13:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:32 smithi105 ceph-mon[46969]: pgmap v251: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 34 MiB/s wr, 1.94k op/s 2024-04-06T14:13:33.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:32 smithi090 ceph-mon[43420]: pgmap v251: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 34 MiB/s wr, 1.94k op/s 2024-04-06T14:13:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:34 smithi134 ceph-mon[47549]: pgmap v252: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 27 MiB/s wr, 1.60k op/s 2024-04-06T14:13:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:34 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:13:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:34 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:13:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:34 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:13:35.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:34 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:13:35.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:34 smithi105 ceph-mon[46969]: pgmap v252: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 27 MiB/s wr, 1.60k op/s 2024-04-06T14:13:35.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:34 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:13:35.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:34 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:13:35.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:34 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:13:35.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:34 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:13:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:34 smithi090 ceph-mon[43420]: pgmap v252: 97 pgs: 97 active+clean; 7.5 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 27 MiB/s wr, 1.60k op/s 2024-04-06T14:13:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:34 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:13:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:34 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:13:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:34 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:13:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:34 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:13:36.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:35 smithi105 ceph-mon[46969]: pgmap v253: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 39 MiB/s wr, 2.16k op/s 2024-04-06T14:13:36.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:35 smithi090 ceph-mon[43420]: pgmap v253: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 39 MiB/s wr, 2.16k op/s 2024-04-06T14:13:36.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:35 smithi134 ceph-mon[47549]: pgmap v253: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 39 MiB/s wr, 2.16k op/s 2024-04-06T14:13:38.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 470.0 s, 1740.6 tps, lat 57.431 ms stddev 86.022 2024-04-06T14:13:38.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:38 smithi105 ceph-mon[46969]: pgmap v254: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.76k op/s 2024-04-06T14:13:39.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:38 smithi090 ceph-mon[43420]: pgmap v254: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.76k op/s 2024-04-06T14:13:39.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:38 smithi134 ceph-mon[47549]: pgmap v254: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.76k op/s 2024-04-06T14:13:41.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:40 smithi105 ceph-mon[46969]: pgmap v255: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.78k op/s 2024-04-06T14:13:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:40 smithi090 ceph-mon[43420]: pgmap v255: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.78k op/s 2024-04-06T14:13:41.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:40 smithi134 ceph-mon[47549]: pgmap v255: 97 pgs: 97 active+clean; 7.6 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 194 KiB/s rd, 30 MiB/s wr, 1.78k op/s 2024-04-06T14:13:42.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:41 smithi105 ceph-mon[46969]: pgmap v256: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 213 KiB/s rd, 40 MiB/s wr, 2.53k op/s 2024-04-06T14:13:42.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:41 smithi090 ceph-mon[43420]: pgmap v256: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 213 KiB/s rd, 40 MiB/s wr, 2.53k op/s 2024-04-06T14:13:42.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:41 smithi134 ceph-mon[47549]: pgmap v256: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 213 KiB/s rd, 40 MiB/s wr, 2.53k op/s 2024-04-06T14:13:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 475.0 s, 1800.6 tps, lat 55.517 ms stddev 67.980 2024-04-06T14:13:45.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:44 smithi134 ceph-mon[47549]: pgmap v257: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 22 MiB/s wr, 1.32k op/s 2024-04-06T14:13:45.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:44 smithi105 ceph-mon[46969]: pgmap v257: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 22 MiB/s wr, 1.32k op/s 2024-04-06T14:13:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:44 smithi090 ceph-mon[43420]: pgmap v257: 97 pgs: 97 active+clean; 7.7 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 193 KiB/s rd, 22 MiB/s wr, 1.32k op/s 2024-04-06T14:13:47.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:46 smithi105 ceph-mon[46969]: pgmap v258: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 256 KiB/s rd, 29 MiB/s wr, 1.70k op/s 2024-04-06T14:13:47.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:46 smithi090 ceph-mon[43420]: pgmap v258: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 256 KiB/s rd, 29 MiB/s wr, 1.70k op/s 2024-04-06T14:13:47.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:46 smithi134 ceph-mon[47549]: pgmap v258: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 256 KiB/s rd, 29 MiB/s wr, 1.70k op/s 2024-04-06T14:13:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 480.0 s, 1870.4 tps, lat 52.975 ms stddev 56.748 2024-04-06T14:13:48.940 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:13:48.940 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:13:48.999 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:48 smithi105 ceph-mon[46969]: pgmap v259: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:49.032 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:48 smithi090 ceph-mon[43420]: pgmap v259: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:49.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:48 smithi134 ceph-mon[47549]: pgmap v259: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:49.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 -- 172.21.15.90:0/3569455861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8010f4b0 msgr2=0x7f9b80107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 --2- 172.21.15.90:0/3569455861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8010f4b0 0x7f9b80107db0 secure :-1 s=READY pgs=658 cs=0 l=1 rev1=1 crypto rx=0x7f9b7c01a9a0 tx=0x7f9b7c01acb0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 -- 172.21.15.90:0/3569455861 shutdown_connections 2024-04-06T14:13:49.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 --2- 172.21.15.90:0/3569455861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b8011a590 0x7f9b8011c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 --2- 172.21.15.90:0/3569455861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8010f4b0 0x7f9b80107db0 unknown :-1 s=CLOSED pgs=658 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 --2- 172.21.15.90:0/3569455861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f920 0x7f9b801082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.181+0000 7f9b885a7700 1 -- 172.21.15.90:0/3569455861 >> 172.21.15.90:0/3569455861 conn(0x7f9b8006be00 msgr2=0x7f9b8006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.182+0000 7f9b885a7700 1 -- 172.21.15.90:0/3569455861 shutdown_connections 2024-04-06T14:13:49.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.182+0000 7f9b885a7700 1 -- 172.21.15.90:0/3569455861 wait complete. 2024-04-06T14:13:49.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 Processor -- start 2024-04-06T14:13:49.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 -- start start 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f4b0 0x7f9b80119a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b8010f920 0x7f9b80119fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b80116550 con 0x7f9b8011a590 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b801166c0 con 0x7f9b8010f4b0 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b885a7700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b80116880 con 0x7f9b8010f920 2024-04-06T14:13:49.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86b44700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86b44700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39760/0 (socket says 172.21.15.90:39760) 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86b44700 1 -- 172.21.15.90:0/2725599751 learned_addr learned my addr 172.21.15.90:0/2725599751 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86343700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f4b0 0x7f9b80119a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86b44700 1 -- 172.21.15.90:0/2725599751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b8010f920 msgr2=0x7f9b80119fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.183+0000 7f9b86b44700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b8010f920 0x7f9b80119fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b86b44700 1 -- 172.21.15.90:0/2725599751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f4b0 msgr2=0x7f9b80119a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b86b44700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f4b0 0x7f9b80119a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b86b44700 1 -- 172.21.15.90:0/2725599751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b7c01a600 con 0x7f9b8011a590 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b86b44700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800b9e0 tx=0x7f9b7800bda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b777fe700 1 -- 172.21.15.90:0/2725599751 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b7800c9b0 con 0x7f9b8011a590 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b801aa220 con 0x7f9b8011a590 2024-04-06T14:13:49.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b801aa6a0 con 0x7f9b8011a590 2024-04-06T14:13:49.191 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b777fe700 1 -- 172.21.15.90:0/2725599751 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b78014440 con 0x7f9b8011a590 2024-04-06T14:13:49.191 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.184+0000 7f9b777fe700 1 -- 172.21.15.90:0/2725599751 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b780125a0 con 0x7f9b8011a590 2024-04-06T14:13:49.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.186+0000 7f9b777fe700 1 -- 172.21.15.90:0/2725599751 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9b78012700 con 0x7f9b8011a590 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.187+0000 7f9b777fe700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 0x7f9b6c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.187+0000 7f9b777fe700 1 -- 172.21.15.90:0/2725599751 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b78006e40 con 0x7f9b8011a590 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 Processor -- start 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b86343700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 0x7f9b6c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 -- start start 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c675b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c6821d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c5cd990 con 0x55c73c675780 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c5cb350 con 0x55c73c6760a0 2024-04-06T14:13:49.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c5c6c40 con 0x55c73c67fde0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b86343700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c675b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57104/0 (socket says 172.21.15.90:57104) 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 -- 172.21.15.90:0/1021431370 learned_addr learned my addr 172.21.15.90:0/1021431370 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b86b44700 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c6821d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.188+0000 7f9b86343700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c675b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39774/0 (socket says 172.21.15.90:39774) 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b86343700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 0x7f9b6c0a37d0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9b7c003800 tx=0x7f9b7c01af10 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 -- 172.21.15.90:0/1021431370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 msgr2=0x55c73c6821d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c6821d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 -- 172.21.15.90:0/1021431370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 msgr2=0x55c73c675b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c675b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 -- 172.21.15.90:0/1021431370 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c73c4483b0 con 0x55c73c6760a0 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b86b44700 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c6821d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:49.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.189+0000 7f9b85b42700 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f9b70009820 tx=0x7f9b70009b30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6dc67dc5cdbe6615 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.190+0000 7f9b74ff9700 1 -- 172.21.15.90:0/1021431370 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b70011830 con 0x55c73c6760a0 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.190+0000 7f9b74ff9700 1 -- 172.21.15.90:0/1021431370 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b70011e50 con 0x55c73c6760a0 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.190+0000 7f9b74ff9700 1 -- 172.21.15.90:0/1021431370 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b7000b360 con 0x55c73c6760a0 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 -- 172.21.15.90:0/1021431370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 msgr2=0x55c73c67f8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f9b70009820 tx=0x7f9b70009b30 comp rx=0 tx=0).stop 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 -- 172.21.15.90:0/1021431370 shutdown_connections 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c6821d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c675b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/1021431370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67f8a0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 -- 172.21.15.90:0/1021431370 >> 172.21.15.90:0/1021431370 conn(0x55c73c5d46a0 msgr2=0x55c73c670a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 -- 172.21.15.90:0/1021431370 shutdown_connections 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.191+0000 7f9b9475bb80 1 -- 172.21.15.90:0/1021431370 wait complete. 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 -- start start 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 Processor -- start 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c67f780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67cb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c562d80 con 0x55c73c675780 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c562a20 con 0x55c73c6760a0 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c44a6a0 con 0x55c73c67fde0 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.192+0000 7f9b9475bb80 10 client.0 _ll_register_callbacks cb 0x55c73c56c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45384/0 (socket says 172.21.15.90:45384) 2024-04-06T14:13:49.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 -- 172.21.15.90:0/290968547 learned_addr learned my addr 172.21.15.90:0/290968547 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b85b42700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67cb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86343700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c67f780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 -- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 msgr2=0x55c73c67cb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67cb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 -- 172.21.15.90:0/290968547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 msgr2=0x55c73c67f780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c67f780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 -- 172.21.15.90:0/290968547 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b801aa6a0 con 0x55c73c67fde0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b86b44700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800b9b0 tx=0x7f9b78099510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f045e5e038d9db58 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.193+0000 7f9b69ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b7809d120 con 0x55c73c67fde0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c73c587380 con 0x55c73c67fde0 2024-04-06T14:13:49.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b780cb070 con 0x55c73c67fde0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b78098a00 con 0x55c73c67fde0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 10 client.26320 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f9b70009480 con 0x55c73c67fde0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c73c587380 con 0x55c73c67fde0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b780b7b00 con 0x55c73c67fde0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 10 client.26320 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 10 client.26320 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 20 client.26320 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c68f940 0x55c73c691d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 4 client.26320 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.194+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c73c5bf700 con 0x55c73c68f940 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.195+0000 7f9b86343700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c68f940 0x55c73c691d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.195+0000 7f9b86343700 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c68f940 0x55c73c691d60 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bc318d13c186d21 server_cookie=305986b149ce78b4 in_seq=0 out_seq=0 2024-04-06T14:13:49.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.195+0000 7f9b69ffb700 10 client.26320 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.195+0000 7f9b69ffb700 1 -- 172.21.15.90:0/290968547 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c73c5bf700 con 0x55c73c68f940 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.195+0000 7f9b69ffb700 10 client.26320 handle_command_reply: tid=0 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 client.26320 shutdown 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 msgr2=0x55c73c67d0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800b9b0 tx=0x7f9b78099510 comp rx=0 tx=0).stop 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 shutdown_connections 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c68f940 0x55c73c691d60 crc :-1 s=READY pgs=153 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c67d0d0 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c675780 0x55c73c67f780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/290968547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c6760a0 0x55c73c67cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 >> 172.21.15.90:0/290968547 conn(0x55c73c5d46a0 msgr2=0x55c73c66fc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b69ffb700 0 client.26320 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:49.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 shutdown_connections 2024-04-06T14:13:49.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 1 -- 172.21.15.90:0/290968547 wait complete. 2024-04-06T14:13:49.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.196+0000 7f9b9475bb80 20 client.26320 trim_cache size 0 max 16384 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 Processor -- start 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 -- start start 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c675780 0x55c73c686ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c67fde0 0x55c73c670a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c67e070 con 0x55c73c6760a0 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c676a10 con 0x55c73c67fde0 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.218+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c668330 con 0x55c73c675780 2024-04-06T14:13:49.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39786/0 (socket says 172.21.15.90:39786) 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 -- 172.21.15.90:0/3087522506 learned_addr learned my addr 172.21.15.90:0/3087522506 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b86343700 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c675780 0x55c73c686ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b86b44700 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c67fde0 0x55c73c670a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 -- 172.21.15.90:0/3087522506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c675780 msgr2=0x55c73c686ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c675780 0x55c73c686ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 -- 172.21.15.90:0/3087522506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c67fde0 msgr2=0x55c73c670a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c67fde0 0x55c73c670a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 -- 172.21.15.90:0/3087522506 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c73c3f4b60 con 0x55c73c6760a0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f9b70016000 tx=0x7f9b70013040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=76ee26d43eb7b92a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b757fa700 1 -- 172.21.15.90:0/3087522506 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b70006ca0 con 0x55c73c6760a0 2024-04-06T14:13:49.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b757fa700 1 -- 172.21.15.90:0/3087522506 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b70006b30 con 0x55c73c6760a0 2024-04-06T14:13:49.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.219+0000 7f9b757fa700 1 -- 172.21.15.90:0/3087522506 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b700119b0 con 0x55c73c6760a0 2024-04-06T14:13:49.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3087522506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 msgr2=0x55c73c670520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 secure :-1 s=READY pgs=660 cs=0 l=1 rev1=1 crypto rx=0x7f9b70016000 tx=0x7f9b70013040 comp rx=0 tx=0).stop 2024-04-06T14:13:49.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3087522506 shutdown_connections 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c67fde0 0x55c73c670a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c675780 0x55c73c686ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3087522506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c670520 unknown :-1 s=CLOSED pgs=660 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3087522506 >> 172.21.15.90:0/3087522506 conn(0x55c73c5d46a0 msgr2=0x55c73c67b420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3087522506 shutdown_connections 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3087522506 wait complete. 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.220+0000 7f9b9475bb80 1 -- start start 2024-04-06T14:13:49.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 Processor -- start 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c675780 0x55c73c6850c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c685b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c686e30 con 0x55c73c6760a0 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c6e3780 con 0x55c73c675780 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c73c6e38c0 con 0x55c73c67fde0 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b9475bb80 10 client.0 _ll_register_callbacks cb 0x55c73c56c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39796/0 (socket says 172.21.15.90:39796) 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 -- 172.21.15.90:0/3524684586 learned_addr learned my addr 172.21.15.90:0/3524684586 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b86b44700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c685b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b86343700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c675780 0x55c73c6850c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 -- 172.21.15.90:0/3524684586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 msgr2=0x55c73c685b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c685b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 -- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c675780 msgr2=0x55c73c6850c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c675780 0x55c73c6850c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.221+0000 7f9b85b42700 1 -- 172.21.15.90:0/3524684586 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c73c4483b0 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.222+0000 7f9b85b42700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f9b70009ff0 tx=0x7f9b700105b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fe35e4e08fb722ef server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.222+0000 7f9b69ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.222+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b70009190 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.222+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b70011630 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.222+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9b70011e50 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b700117f0 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 10 client.16779 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c73c58cea0 con 0x55c73c6760a0 2024-04-06T14:13:49.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b7003c030 con 0x55c73c6760a0 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b7002fbb0 con 0x55c73c6760a0 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 10 client.16779 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 10 client.16779 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 20 client.16779 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c6e4d00 0x55c73c6e7120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 4 client.16779 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c73c415cf0 con 0x55c73c6e4d00 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.223+0000 7f9b86343700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c6e4d00 0x55c73c6e7120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b69ffb700 10 client.16779 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b86343700 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c6e4d00 0x55c73c6e7120 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9d77e2b55dacd608 server_cookie=4085219b46af5095 in_seq=0 out_seq=0 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b69ffb700 1 -- 172.21.15.90:0/3524684586 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55c73c415cf0 con 0x55c73c6e4d00 2024-04-06T14:13:49.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b69ffb700 10 client.16779 handle_command_reply: tid=0 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 client.16779 shutdown 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 msgr2=0x55c73c685600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 secure :-1 s=READY pgs=661 cs=0 l=1 rev1=1 crypto rx=0x7f9b70009ff0 tx=0x7f9b700105b0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 shutdown_connections 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c73c6e4d00 0x55c73c6e7120 crc :-1 s=READY pgs=154 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c73c67fde0 0x55c73c685b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c73c675780 0x55c73c6850c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 --2- 172.21.15.90:0/3524684586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c73c6760a0 0x55c73c685600 unknown :-1 s=CLOSED pgs=661 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 >> 172.21.15.90:0/3524684586 conn(0x55c73c5d46a0 msgr2=0x55c73c67b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b69ffb700 0 client.16779 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.224+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 shutdown_connections 2024-04-06T14:13:49.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.225+0000 7f9b9475bb80 1 -- 172.21.15.90:0/3524684586 wait complete. 2024-04-06T14:13:49.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.225+0000 7f9b9475bb80 20 client.16779 trim_cache size 0 max 16384 2024-04-06T14:13:49.231 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:49.231 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:13:49.231 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:13:49.231 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:49.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 msgr2=0x7f9b6c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 0x7f9b6c0a37d0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f9b7c003800 tx=0x7f9b7c01af10 comp rx=0 tx=0).stop 2024-04-06T14:13:49.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 msgr2=0x7f9b80115d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 secure :-1 s=READY pgs=659 cs=0 l=1 rev1=1 crypto rx=0x7f9b7800b9e0 tx=0x7f9b7800bda0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 shutdown_connections 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b6c0a1310 0x7f9b6c0a37d0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b8011a590 0x7f9b80115d80 unknown :-1 s=CLOSED pgs=659 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b8010f4b0 0x7f9b80119a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 --2- 172.21.15.90:0/2725599751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b8010f920 0x7f9b80119fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.227+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 >> 172.21.15.90:0/2725599751 conn(0x7f9b8006be00 msgr2=0x7f9b80108f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.228+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 shutdown_connections 2024-04-06T14:13:49.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.228+0000 7f9b885a7700 1 -- 172.21.15.90:0/2725599751 wait complete. 2024-04-06T14:13:49.244 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:13:49.244 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:13:49.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 -- 172.21.15.90:0/1230965943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c4104600 msgr2=0x7f70c410c180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 --2- 172.21.15.90:0/1230965943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c4104600 0x7f70c410c180 secure :-1 s=READY pgs=662 cs=0 l=1 rev1=1 crypto rx=0x7f70bc00c270 tx=0x7f70bc00c580 comp rx=0 tx=0).stop 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 -- 172.21.15.90:0/1230965943 shutdown_connections 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 --2- 172.21.15.90:0/1230965943 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c4104600 0x7f70c410c180 unknown :-1 s=CLOSED pgs=662 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 --2- 172.21.15.90:0/1230965943 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c40ff8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 --2- 172.21.15.90:0/1230965943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70c40ffe90 0x7f70c4103ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.420+0000 7f70c9a82700 1 -- 172.21.15.90:0/1230965943 >> 172.21.15.90:0/1230965943 conn(0x7f70c40760a0 msgr2=0x7f70c40fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.422+0000 7f70c9a82700 1 -- 172.21.15.90:0/1230965943 shutdown_connections 2024-04-06T14:13:49.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.422+0000 7f70c9a82700 1 -- 172.21.15.90:0/1230965943 wait complete. 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.422+0000 7f70c9a82700 1 Processor -- start 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 -- start start 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70c4104600 0x7f70c406c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c406cf60 0x7f70c406d410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70c41b26c0 con 0x7f70c406cf60 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70c41b2830 con 0x7f70c4104600 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c9a82700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f70c41b29f0 con 0x7f70c40ff4e0 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45424/0 (socket says 172.21.15.90:45424) 2024-04-06T14:13:49.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 -- 172.21.15.90:0/2317715341 learned_addr learned my addr 172.21.15.90:0/2317715341 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c3fff700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c406cf60 0x7f70c406d410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 -- 172.21.15.90:0/2317715341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70c4104600 msgr2=0x7f70c406c9d0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70c4104600 0x7f70c406c9d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 -- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c406cf60 msgr2=0x7f70c406d410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c406cf60 0x7f70c406d410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.423+0000 7f70c37fe700 1 -- 172.21.15.90:0/2317715341 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f70bc00ae10 con 0x7f70c40ff4e0 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c37fe700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f70ac00d950 tx=0x7f70ac00dd10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70ac009b40 con 0x7f70c40ff4e0 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f70ac009ca0 con 0x7f70c40ff4e0 2024-04-06T14:13:49.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f70ac00f620 con 0x7f70c40ff4e0 2024-04-06T14:13:49.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f70c41b2cd0 con 0x7f70c40ff4e0 2024-04-06T14:13:49.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.424+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f70c41b3190 con 0x7f70c40ff4e0 2024-04-06T14:13:49.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.425+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f70c41078d0 con 0x7f70c40ff4e0 2024-04-06T14:13:49.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.427+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f70ac004020 con 0x7f70c40ff4e0 2024-04-06T14:13:49.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.428+0000 7f70c0ff9700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 0x7f70b00a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:49.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.428+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f70ac0ccf10 con 0x7f70c40ff4e0 2024-04-06T14:13:49.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.428+0000 7f70c2ffd700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 0x7f70b00a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:49.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.429+0000 7f70c2ffd700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 0x7f70b00a3810 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f70c406e120 tx=0x7f70b400b3f0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:49.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.431+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f70ac096340 con 0x7f70c40ff4e0 2024-04-06T14:13:49.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.798+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f70c41009b0 con 0x7f70c40ff4e0 2024-04-06T14:13:49.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.802+0000 7f70c0ff9700 1 -- 172.21.15.90:0/2317715341 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f70ac016090 con 0x7f70c40ff4e0 2024-04-06T14:13:49.808 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:13:49.809 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:13:49.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.813+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 msgr2=0x7f70b00a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.813+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 0x7f70b00a3810 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f70c406e120 tx=0x7f70b400b3f0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.813+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 msgr2=0x7f70c4072950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:49.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.813+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f70ac00d950 tx=0x7f70ac00dd10 comp rx=0 tx=0).stop 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 shutdown_connections 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f70b00a1350 0x7f70b00a3810 secure :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f70c406e120 tx=0x7f70b400b3f0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f70c406cf60 0x7f70c406d410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f70c40ff4e0 0x7f70c4072950 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 --2- 172.21.15.90:0/2317715341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f70c4104600 0x7f70c406c9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:49.815 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.814+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 >> 172.21.15.90:0/2317715341 conn(0x7f70c40760a0 msgr2=0x7f70c4102240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:49.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.816+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 shutdown_connections 2024-04-06T14:13:49.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:49.816+0000 7f70c9a82700 1 -- 172.21.15.90:0/2317715341 wait complete. 2024-04-06T14:13:49.818 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:13:49.835 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:49.835 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:13:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 -- 172.21.15.90:0/14613734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc104f70 msgr2=0x7f3bfc10caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/14613734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10caf0 secure :-1 s=READY pgs=663 cs=0 l=1 rev1=1 crypto rx=0x7f3bf400b0a0 tx=0x7f3bf400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 -- 172.21.15.90:0/14613734 shutdown_connections 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/14613734 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10caf0 unknown :-1 s=CLOSED pgs=663 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/14613734 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc0ffe70 0x7f3bfc100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/14613734 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc100820 0x7f3bfc104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 -- 172.21.15.90:0/14613734 >> 172.21.15.90:0/14613734 conn(0x7f3bfc076d50 msgr2=0x7f3bfc077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 -- 172.21.15.90:0/14613734 shutdown_connections 2024-04-06T14:13:50.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.000+0000 7f3c03c0b700 1 -- 172.21.15.90:0/14613734 wait complete. 2024-04-06T14:13:50.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.001+0000 7f3c03c0b700 1 Processor -- start 2024-04-06T14:13:50.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.001+0000 7f3c03c0b700 1 -- start start 2024-04-06T14:13:50.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc0ffe70 0x7f3bfc10d430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc10deb0 0x7f3bfc1a5500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bfc1a5ad0 con 0x7f3bfc0ffe70 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bfc1a5c10 con 0x7f3bfc10deb0 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c03c0b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bfc1a5dd0 con 0x7f3bfc104f70 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:45438/0 (socket says 172.21.15.90:45438) 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 -- 172.21.15.90:0/3455447234 learned_addr learned my addr 172.21.15.90:0/3455447234 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c021a8700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc10deb0 0x7f3bfc1a5500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 -- 172.21.15.90:0/3455447234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc10deb0 msgr2=0x7f3bfc1a5500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc10deb0 0x7f3bfc1a5500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 -- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc0ffe70 msgr2=0x7f3bfc10d430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc0ffe70 0x7f3bfc10d430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 -- 172.21.15.90:0/3455447234 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3bf8009520 con 0x7f3bfc104f70 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.002+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f3bec00e970 tx=0x7f3bec00ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.003+0000 7f3bf2ffd700 1 -- 172.21.15.90:0/3455447234 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec009790 con 0x7f3bfc104f70 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.003+0000 7f3bf2ffd700 1 -- 172.21.15.90:0/3455447234 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3bec00eec0 con 0x7f3bfc104f70 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.003+0000 7f3bf2ffd700 1 -- 172.21.15.90:0/3455447234 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec005590 con 0x7f3bfc104f70 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.003+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3bf4009d00 con 0x7f3bfc104f70 2024-04-06T14:13:50.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.003+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3bfc1a63e0 con 0x7f3bfc104f70 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.007+0000 7f3bf2ffd700 1 -- 172.21.15.90:0/3455447234 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3bec00cbd0 con 0x7f3bfc104f70 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.007+0000 7f3c0fdbfb80 1 Processor -- start 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.008+0000 7f3bf2ffd700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 0x7f3be80a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.008+0000 7f3bf2ffd700 1 -- 172.21.15.90:0/3455447234 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3bec014070 con 0x7f3bfc104f70 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.008+0000 7f3c0fdbfb80 1 -- start start 2024-04-06T14:13:50.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.008+0000 7f3c021a8700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 0x7f3be80a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.008+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a3cb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a491d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48994990 con 0x563a48a3d0a0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48992350 con 0x563a48a46de0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a4898dc40 con 0x563a48a3c780 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39822/0 (socket says 172.21.15.90:39822) 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 -- 172.21.15.90:0/3559266166 learned_addr learned my addr 172.21.15.90:0/3559266166 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c019a7700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a3cb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c021a8700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a491d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 -- 172.21.15.90:0/3559266166 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 msgr2=0x563a48a3cb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a3cb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 -- 172.21.15.90:0/3559266166 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 msgr2=0x563a48a491d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a491d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 -- 172.21.15.90:0/3559266166 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563a4880f3b0 con 0x563a48a3d0a0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c019a7700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a3cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c021a8700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 0x7f3be80a37d0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f3bf4009fd0 tx=0x7f3bf4006210 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f3bec009230 tx=0x7f3bec09a700 comp rx=0 tx=0).ready entity=mon.0 client_cookie=451d696a0775116 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3be7fff700 1 -- 172.21.15.90:0/3559266166 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec0a2480 con 0x563a48a3d0a0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3be7fff700 1 -- 172.21.15.90:0/3559266166 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3bec0ca8c0 con 0x563a48a3d0a0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.009+0000 7f3be7fff700 1 -- 172.21.15.90:0/3559266166 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec09a460 con 0x563a48a3d0a0 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3559266166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 msgr2=0x563a48a468a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 secure :-1 s=READY pgs=664 cs=0 l=1 rev1=1 crypto rx=0x7f3bec009230 tx=0x7f3bec09a700 comp rx=0 tx=0).stop 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3559266166 shutdown_connections 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a491d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a3cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3559266166 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a468a0 unknown :-1 s=CLOSED pgs=664 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3559266166 >> 172.21.15.90:0/3559266166 conn(0x563a4899b6a0 msgr2=0x563a48a37a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3559266166 shutdown_connections 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.010+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3559266166 wait complete. 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 -- start start 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 Processor -- start 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 0x563a48a46860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 0x563a48a44240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48929d80 con 0x563a48a3d0a0 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48929a20 con 0x563a48a3c780 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a488116a0 con 0x563a48a46de0 2024-04-06T14:13:50.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c0fdbfb80 10 client.0 _ll_register_callbacks cb 0x563a48933470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c021a8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 0x563a48a44240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39834/0 (socket says 172.21.15.90:39834) 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c011a6700 1 -- 172.21.15.90:0/1881456804 learned_addr learned my addr 172.21.15.90:0/1881456804 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.011+0000 7f3c019a7700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 0x563a48a46860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 -- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 msgr2=0x563a48a46860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 0x563a48a46860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 -- 172.21.15.90:0/1881456804 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 msgr2=0x563a48a43b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 -- 172.21.15.90:0/1881456804 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3bfc1a63e0 con 0x563a48a46de0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c019a7700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 0x563a48a46860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c011a6700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c021a8700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 0x563a48a44240 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f3bf400b780 tx=0x7f3bf400b3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7efbc8530c4c6d23 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3be57fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:13:50.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bf400e040 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3bf4003e10 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563a4894e380 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.012+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3bf4009440 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 10 client.26344 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563a487bbb60 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563a4894e380 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3bf402f020 con 0x563a48a46de0 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 10 client.26344 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 10 client.26344 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 20 client.26344 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48a4da50 0x563a48a4fe70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 4 client.26344 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563a48986700 con 0x563a48a4da50 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.013+0000 7f3c019a7700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48a4da50 0x563a48a4fe70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.014+0000 7f3be57fa700 10 client.26344 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.014+0000 7f3c019a7700 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48a4da50 0x563a48a4fe70 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=6fdee034e329a951 server_cookie=f66d565baecfd981 in_seq=0 out_seq=0 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.014+0000 7f3be57fa700 1 -- 172.21.15.90:0/1881456804 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563a48986700 con 0x563a48a4da50 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.014+0000 7f3be57fa700 10 client.26344 handle_command_reply: tid=0 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 client.26344 shutdown 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 msgr2=0x563a48a44240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 0x563a48a44240 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f3bf400b780 tx=0x7f3bf400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 shutdown_connections 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48a4da50 0x563a48a4fe70 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a46de0 0x563a48a44240 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3be57fa700 0 client.26344 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3c780 0x563a48a46860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/1881456804 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3d0a0 0x563a48a43b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 >> 172.21.15.90:0/1881456804 conn(0x563a4899b6a0 msgr2=0x563a48a36c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 shutdown_connections 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/1881456804 wait complete. 2024-04-06T14:13:50.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.015+0000 7f3c0fdbfb80 20 client.26344 trim_cache size 0 max 16384 2024-04-06T14:13:50.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.029+0000 7f3c0fdbfb80 1 Processor -- start 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.029+0000 7f3c0fdbfb80 1 -- start start 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.029+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.029+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3d0a0 0x563a48a547a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a42420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48a5b150 con 0x563a48a3c780 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48a54ce0 con 0x563a48a46de0 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48a504d0 con 0x563a48a3d0a0 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39850/0 (socket says 172.21.15.90:39850) 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 -- 172.21.15.90:0/2156037023 learned_addr learned my addr 172.21.15.90:0/2156037023 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c021a8700 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a42420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c011a6700 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3d0a0 0x563a48a547a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 -- 172.21.15.90:0/2156037023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3d0a0 msgr2=0x563a48a547a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3d0a0 0x563a48a547a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 -- 172.21.15.90:0/2156037023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 msgr2=0x563a48a42420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a42420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 -- 172.21.15.90:0/2156037023 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563a48953ea0 con 0x563a48a3c780 2024-04-06T14:13:50.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.030+0000 7f3c019a7700 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x563a48a50740 tx=0x7f3bf80099d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1cda2d107c20d784 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.031+0000 7f3bf0da5700 1 -- 172.21.15.90:0/2156037023 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bf8018030 con 0x563a48a3c780 2024-04-06T14:13:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.031+0000 7f3bf0da5700 1 -- 172.21.15.90:0/2156037023 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3bf8019040 con 0x563a48a3c780 2024-04-06T14:13:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.031+0000 7f3bf0da5700 1 -- 172.21.15.90:0/2156037023 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bf801ab90 con 0x563a48a3c780 2024-04-06T14:13:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.032+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/2156037023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 msgr2=0x563a48a441a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.032+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 secure :-1 s=READY pgs=665 cs=0 l=1 rev1=1 crypto rx=0x563a48a50740 tx=0x7f3bf80099d0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/2156037023 shutdown_connections 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a46de0 0x563a48a42420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a3c780 0x563a48a441a0 unknown :-1 s=CLOSED pgs=665 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/2156037023 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3d0a0 0x563a48a547a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/2156037023 >> 172.21.15.90:0/2156037023 conn(0x563a4899b6a0 msgr2=0x563a48a70350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/2156037023 shutdown_connections 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.033+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/2156037023 wait complete. 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 -- start start 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 Processor -- start 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a54760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a46de0 0x563a48a534e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48a53bf0 con 0x563a48a46de0 2024-04-06T14:13:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48abb400 con 0x563a48a3d0a0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563a48abb540 con 0x563a48a3c780 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c0fdbfb80 10 client.0 _ll_register_callbacks cb 0x7f3bf4008f60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c011a6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57212/0 (socket says 172.21.15.90:57212) 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c011a6700 1 -- 172.21.15.90:0/3972545729 learned_addr learned my addr 172.21.15.90:0/3972545729 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.034+0000 7f3c021a8700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a46de0 0x563a48a534e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c019a7700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a54760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 -- 172.21.15.90:0/3972545729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 msgr2=0x563a48a54760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a54760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 -- 172.21.15.90:0/3972545729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a46de0 msgr2=0x563a48a534e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a46de0 0x563a48a534e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 -- 172.21.15.90:0/3972545729 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3bf8009b60 con 0x563a48a3d0a0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c011a6700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f3bec099f90 tx=0x7f3bec09def0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3eda08183d2ba45a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3be57fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bec09a460 con 0x563a48a3d0a0 2024-04-06T14:13:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3bec0a2480 con 0x563a48a3d0a0 2024-04-06T14:13:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3bec0ca8c0 con 0x563a48a3d0a0 2024-04-06T14:13:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.035+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3bec0b83f0 con 0x563a48a3d0a0 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.036+0000 7f3c0fdbfb80 10 client.26513 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.036+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563a48813cd0 con 0x563a48a3d0a0 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3bec0ca8c0 con 0x563a48a3d0a0 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3bec098f60 con 0x563a48a3d0a0 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 10 client.26513 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 10 client.26513 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 20 client.26513 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48abc740 0x563a48abeb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 4 client.26513 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x563a487dccf0 con 0x563a48abc740 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.037+0000 7f3c019a7700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48abc740 0x563a48abeb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3c019a7700 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48abc740 0x563a48abeb30 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=782ba81513a31894 server_cookie=e29dd87f476d4c4c in_seq=0 out_seq=0 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3be57fa700 10 client.26513 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3be57fa700 1 -- 172.21.15.90:0/3972545729 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x563a487dccf0 con 0x563a48abc740 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3be57fa700 10 client.26513 handle_command_reply: tid=0 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3c0fdbfb80 1 client.26513 shutdown 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 msgr2=0x563a48a52da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.038+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f3bec099f90 tx=0x7f3bec09def0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 shutdown_connections 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x563a48abc740 0x563a48abeb30 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563a48a46de0 0x563a48a534e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563a48a3c780 0x563a48a54760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 --2- 172.21.15.90:0/3972545729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563a48a3d0a0 0x563a48a52da0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 >> 172.21.15.90:0/3972545729 conn(0x563a4899b6a0 msgr2=0x563a48a42360 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3be57fa700 0 client.26513 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:13:50.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 shutdown_connections 2024-04-06T14:13:50.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.039+0000 7f3c0fdbfb80 1 -- 172.21.15.90:0/3972545729 wait complete. 2024-04-06T14:13:50.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c0fdbfb80 20 client.26513 trim_cache size 0 max 16384 2024-04-06T14:13:50.042 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 msgr2=0x7f3be80a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 0x7f3be80a37d0 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f3bf4009fd0 tx=0x7f3bf4006210 comp rx=0 tx=0).stop 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 msgr2=0x7f3bfc10d970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f3bec00e970 tx=0x7f3bec00ec80 comp rx=0 tx=0).stop 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 shutdown_connections 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3be80a1310 0x7f3be80a37d0 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3bfc10deb0 0x7f3bfc1a5500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3bfc0ffe70 0x7f3bfc10d430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 --2- 172.21.15.90:0/3455447234 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3bfc104f70 0x7f3bfc10d970 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 >> 172.21.15.90:0/3455447234 conn(0x7f3bfc076d50 msgr2=0x7f3bfc0fdd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 shutdown_connections 2024-04-06T14:13:50.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.041+0000 7f3c03c0b700 1 -- 172.21.15.90:0/3455447234 wait complete. 2024-04-06T14:13:50.059 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:50.059 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:13:50.086 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:49 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2317715341' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:50.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:49 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2317715341' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:50.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:49 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2317715341' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 -- 172.21.15.90:0/4044472815 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ff220 msgr2=0x7fefcc0ff6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 --2- 172.21.15.90:0/4044472815 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ff220 0x7fefcc0ff6a0 secure :-1 s=READY pgs=666 cs=0 l=1 rev1=1 crypto rx=0x7fefc0009990 tx=0x7fefc0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 -- 172.21.15.90:0/4044472815 shutdown_connections 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 --2- 172.21.15.90:0/4044472815 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc10dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 --2- 172.21.15.90:0/4044472815 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc105d00 0x7fefcc1060e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 --2- 172.21.15.90:0/4044472815 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ff220 0x7fefcc0ff6a0 unknown :-1 s=CLOSED pgs=666 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 -- 172.21.15.90:0/4044472815 >> 172.21.15.90:0/4044472815 conn(0x7fefcc0761b0 msgr2=0x7fefcc0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 -- 172.21.15.90:0/4044472815 shutdown_connections 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.246+0000 7fefd24b9700 1 -- 172.21.15.90:0/4044472815 wait complete. 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 Processor -- start 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 -- start start 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc0ff220 0x7fefcc197230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc105d00 0x7fefcc19bb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc19c150 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc19c2c0 con 0x7fefcc0ff220 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.247+0000 7fefd24b9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fefcc19c480 con 0x7fefcc105d00 2024-04-06T14:13:50.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39874/0 (socket says 172.21.15.90:39874) 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 -- 172.21.15.90:0/2239744954 learned_addr learned my addr 172.21.15.90:0/2239744954 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefd0a56700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc105d00 0x7fefcc19bb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcbfff700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc0ff220 0x7fefcc197230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 -- 172.21.15.90:0/2239744954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc105d00 msgr2=0x7fefcc19bb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc105d00 0x7fefcc19bb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 -- 172.21.15.90:0/2239744954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc0ff220 msgr2=0x7fefcc197230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc0ff220 0x7fefcc197230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 -- 172.21.15.90:0/2239744954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fefc00095f0 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fefc0005770 tx=0x7fefc00057a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.248+0000 7fefc97fa700 1 -- 172.21.15.90:0/2239744954 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc001d070 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.249+0000 7fefc97fa700 1 -- 172.21.15.90:0/2239744954 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefc0021410 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.249+0000 7fefc97fa700 1 -- 172.21.15.90:0/2239744954 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc0017650 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.249+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fefcc19c700 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.249+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefcc100510 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.251+0000 7fefc97fa700 1 -- 172.21.15.90:0/2239744954 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fefc00217f0 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.252+0000 7fefc97fa700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 0x7fefb40a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.252+0000 7fefde66db80 1 Processor -- start 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.252+0000 7fefde66db80 1 -- start start 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.252+0000 7fefc97fa700 1 -- 172.21.15.90:0/2239744954 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fefc00cdb30 con 0x7fefcc0ffbe0 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12a8b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12b28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b51d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda1200990 con 0x55cda12a8780 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda11fe350 con 0x55cda12b2de0 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefde66db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda11f9c40 con 0x55cda12a90a0 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcbfff700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 0x7fefb40a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12a8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12b28a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12a8b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39882/0 (socket says 172.21.15.90:39882) 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcbfff700 1 -- 172.21.15.90:0/985513290 learned_addr learned my addr 172.21.15.90:0/985513290 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b51d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 -- 172.21.15.90:0/985513290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 msgr2=0x55cda12b28a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12b28a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 -- 172.21.15.90:0/985513290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 msgr2=0x55cda12a8b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12a8b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 -- 172.21.15.90:0/985513290 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cda107b3b0 con 0x55cda12b2de0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefcbfff700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 0x7fefb40a3750 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fefbc012a00 tx=0x7fefbc009ba0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefd0a56700 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b51d0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fefc4010820 tx=0x7fefc400b040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=132c6ab24570846b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.253+0000 7fefb27fc700 1 -- 172.21.15.90:0/985513290 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc4015050 con 0x55cda12b2de0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.254+0000 7fefb27fc700 1 -- 172.21.15.90:0/985513290 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefc4016040 con 0x55cda12b2de0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.254+0000 7fefb27fc700 1 -- 172.21.15.90:0/985513290 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc4007500 con 0x55cda12b2de0 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 -- 172.21.15.90:0/985513290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 msgr2=0x55cda12b51d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b51d0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fefc4010820 tx=0x7fefc400b040 comp rx=0 tx=0).stop 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 -- 172.21.15.90:0/985513290 shutdown_connections 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b51d0 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12a8b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 --2- 172.21.15.90:0/985513290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12b28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.255+0000 7fefde66db80 1 -- 172.21.15.90:0/985513290 >> 172.21.15.90:0/985513290 conn(0x55cda12076a0 msgr2=0x55cda12a3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.256+0000 7fefde66db80 1 -- 172.21.15.90:0/985513290 shutdown_connections 2024-04-06T14:13:50.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.257+0000 7fefde66db80 1 -- 172.21.15.90:0/985513290 wait complete. 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.257+0000 7fefde66db80 1 -- start start 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.257+0000 7fefde66db80 1 Processor -- start 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12b28a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12afb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda1195d80 con 0x55cda12a8780 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda1195a20 con 0x55cda12b2de0 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda107d6a0 con 0x55cda12a90a0 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefde66db80 10 client.0 _ll_register_callbacks cb 0x55cda119f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57246/0 (socket says 172.21.15.90:57246) 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 -- 172.21.15.90:0/3664205556 learned_addr learned my addr 172.21.15.90:0/3664205556 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12afb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 -- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 msgr2=0x55cda12afb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12afb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 -- 172.21.15.90:0/3664205556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 msgr2=0x55cda12b28a0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12b28a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.258+0000 7fefd0a56700 1 -- 172.21.15.90:0/3664205556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fefc40104d0 con 0x55cda12b2de0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fefc4009960 tx=0x7fefc4004860 comp rx=0 tx=0).ready entity=mon.2 client_cookie=76f54f3cea76e0e6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefab7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc40093a0 con 0x55cda12b2de0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fefc4003e60 con 0x55cda12b2de0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cda11ba380 con 0x55cda12b2de0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fefc4015020 con 0x55cda12b2de0 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefde66db80 10 client.26525 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cda1027b60 con 0x55cda12b2de0 2024-04-06T14:13:50.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.259+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefc4031030 con 0x55cda12b2de0 2024-04-06T14:13:50.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fefc4007bc0 con 0x55cda12b2de0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 10 client.26525 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 10 client.26525 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 20 client.26525 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda12c29e0 0x55cda12c4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 4 client.26525 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cda11f2700 con 0x55cda12c29e0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefcbfff700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda12c29e0 0x55cda12c4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefcbfff700 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda12c29e0 0x55cda12c4dd0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c7c4571537544878 server_cookie=5be89c4c9939fa22 in_seq=0 out_seq=0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.260+0000 7fefab7fe700 10 client.26525 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.261+0000 7fefab7fe700 1 -- 172.21.15.90:0/3664205556 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cda11f2700 con 0x55cda12c29e0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.261+0000 7fefab7fe700 10 client.26525 handle_command_reply: tid=0 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 client.26525 shutdown 2024-04-06T14:13:50.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 msgr2=0x55cda12b02d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fefc4009960 tx=0x7fefc4004860 comp rx=0 tx=0).stop 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 shutdown_connections 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda12c29e0 0x55cda12c4dd0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b02d0 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda12b28a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 --2- 172.21.15.90:0/3664205556 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12afb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefab7fe700 0 client.26525 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 >> 172.21.15.90:0/3664205556 conn(0x55cda12076a0 msgr2=0x55cda12a2c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 shutdown_connections 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 1 -- 172.21.15.90:0/3664205556 wait complete. 2024-04-06T14:13:50.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.262+0000 7fefde66db80 20 client.26525 trim_cache size 0 max 16384 2024-04-06T14:13:50.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 Processor -- start 2024-04-06T14:13:50.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 -- start start 2024-04-06T14:13:50.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda129f890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12af420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b9a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12a4650 con 0x55cda12a8780 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12aa650 con 0x55cda12b2de0 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefde66db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12af960 con 0x55cda12a90a0 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefcbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda129f890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefcbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda129f890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39890/0 (socket says 172.21.15.90:39890) 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefd0a56700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b9a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefcbfff700 1 -- 172.21.15.90:0/3619300850 learned_addr learned my addr 172.21.15.90:0/3619300850 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12af420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefd0a56700 1 -- 172.21.15.90:0/3619300850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 msgr2=0x55cda12af420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12af420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.276+0000 7fefd0a56700 1 -- 172.21.15.90:0/3619300850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 msgr2=0x55cda129f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda129f890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefd0a56700 1 -- 172.21.15.90:0/3619300850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cda11bfea0 con 0x55cda12b2de0 2024-04-06T14:13:50.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefd0a56700 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b9a60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fefc4010e10 tx=0x7fefc4005180 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3cab04c40ce06e2f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefb2ffd700 1 -- 172.21.15.90:0/3619300850 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc4015070 con 0x55cda12b2de0 2024-04-06T14:13:50.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefb2ffd700 1 -- 172.21.15.90:0/3619300850 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefc4031030 con 0x55cda12b2de0 2024-04-06T14:13:50.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.277+0000 7fefb2ffd700 1 -- 172.21.15.90:0/3619300850 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc4003df0 con 0x55cda12b2de0 2024-04-06T14:13:50.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 -- 172.21.15.90:0/3619300850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 msgr2=0x55cda12b9a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b9a60 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fefc4010e10 tx=0x7fefc4005180 comp rx=0 tx=0).stop 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 -- 172.21.15.90:0/3619300850 shutdown_connections 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12b2de0 0x55cda12b9a60 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12a8780 0x55cda129f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 --2- 172.21.15.90:0/3619300850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a90a0 0x55cda12af420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 -- 172.21.15.90:0/3619300850 >> 172.21.15.90:0/3619300850 conn(0x55cda12076a0 msgr2=0x55cda12c9770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 -- 172.21.15.90:0/3619300850 shutdown_connections 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.278+0000 7fefde66db80 1 -- 172.21.15.90:0/3619300850 wait complete. 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 -- start start 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 Processor -- start 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a8780 0x55cda12b97a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12b2de0 0x55cda12b80d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12b8610 con 0x55cda12b2de0 2024-04-06T14:13:50.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12b8780 con 0x55cda12a90a0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cda12b8940 con 0x55cda12a8780 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefde66db80 10 client.0 _ll_register_callbacks cb 0x55cda119f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefcb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefcb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57264/0 (socket says 172.21.15.90:57264) 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.279+0000 7fefcb7fe700 1 -- 172.21.15.90:0/706424215 learned_addr learned my addr 172.21.15.90:0/706424215 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefd0a56700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12b2de0 0x55cda12b80d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 -- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a8780 msgr2=0x55cda12b97a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a8780 0x55cda12b97a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 -- 172.21.15.90:0/706424215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12b2de0 msgr2=0x55cda12b80d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12b2de0 0x55cda12b80d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 -- 172.21.15.90:0/706424215 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55cda1027b60 con 0x55cda12a90a0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefcb7fe700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fefc00976c0 tx=0x7fefc00cb760 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d348f860ea34d841 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefab7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:50.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fefc009b0b0 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fefc009ea80 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fefc4016040 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fefc00a3040 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefde66db80 10 client.26531 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cda107fcd0 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.280+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fefc4016040 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fefc0099bb0 con 0x55cda12a90a0 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 10 client.26531 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 10 client.26531 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 20 client.26531 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda1320110 0x55cda1322530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 4 client.26531 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:50.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55cda1048cf0 con 0x55cda1320110 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefcbfff700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda1320110 0x55cda1322530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefcbfff700 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda1320110 0x55cda1322530 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=11e9d603d61c3d2f server_cookie=931cead9d28bd376 in_seq=0 out_seq=0 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefab7fe700 10 client.26531 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.281+0000 7fefab7fe700 1 -- 172.21.15.90:0/706424215 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55cda1048cf0 con 0x55cda1320110 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefab7fe700 10 client.26531 handle_command_reply: tid=0 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 client.26531 shutdown 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 msgr2=0x55cda12b7b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fefc00976c0 tx=0x7fefc00cb760 comp rx=0 tx=0).stop 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 shutdown_connections 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55cda1320110 0x55cda1322530 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cda12b2de0 0x55cda12b80d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefab7fe700 0 client.26531 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:13:50.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cda12a8780 0x55cda12b97a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 --2- 172.21.15.90:0/706424215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cda12a90a0 0x55cda12b7b90 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 >> 172.21.15.90:0/706424215 conn(0x55cda12076a0 msgr2=0x55cda12a34d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 shutdown_connections 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 1 -- 172.21.15.90:0/706424215 wait complete. 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.282+0000 7fefde66db80 20 client.26531 trim_cache size 0 max 16384 2024-04-06T14:13:50.287 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:50.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 msgr2=0x7fefb40a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 0x7fefb40a3750 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fefbc012a00 tx=0x7fefbc009ba0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 msgr2=0x7fefcc197770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 secure :-1 s=READY pgs=667 cs=0 l=1 rev1=1 crypto rx=0x7fefc0005770 tx=0x7fefc00057a0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 shutdown_connections 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fefb40a1290 0x7fefb40a3750 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fefcc105d00 0x7fefcc19bb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fefcc0ff220 0x7fefcc197230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 --2- 172.21.15.90:0/2239744954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fefcc0ffbe0 0x7fefcc197770 unknown :-1 s=CLOSED pgs=667 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 >> 172.21.15.90:0/2239744954 conn(0x7fefcc0761b0 msgr2=0x7fefcc103ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 shutdown_connections 2024-04-06T14:13:50.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.283+0000 7fefd24b9700 1 -- 172.21.15.90:0/2239744954 wait complete. 2024-04-06T14:13:50.301 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:13:50.301 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 -- 172.21.15.90:0/749309429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 msgr2=0x7ff8b40ffb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 --2- 172.21.15.90:0/749309429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b40ffb90 secure :-1 s=READY pgs=668 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4009990 tx=0x7ff8a4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 -- 172.21.15.90:0/749309429 shutdown_connections 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 --2- 172.21.15.90:0/749309429 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b40ffb90 unknown :-1 s=CLOSED pgs=668 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 --2- 172.21.15.90:0/749309429 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b4104320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 --2- 172.21.15.90:0/749309429 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b40ff650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 -- 172.21.15.90:0/749309429 >> 172.21.15.90:0/749309429 conn(0x7ff8b4074b50 msgr2=0x7ff8b4074f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 -- 172.21.15.90:0/749309429 shutdown_connections 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.527+0000 7ff8bae35700 1 -- 172.21.15.90:0/749309429 wait complete. 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 Processor -- start 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 -- start start 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b4197650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b419b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419c030 con 0x7ff8b4067270 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419c1a0 con 0x7ff8b4103f40 2024-04-06T14:13:50.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8bae35700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8b419c360 con 0x7ff8b40ff1d0 2024-04-06T14:13:50.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39930/0 (socket says 172.21.15.90:39930) 2024-04-06T14:13:50.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/3796141321 learned_addr learned my addr 172.21.15.90:0/3796141321 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.528+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b419b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b4197650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/3796141321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 msgr2=0x7ff8b4197650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b4197650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/3796141321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 msgr2=0x7ff8b419b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b419b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/3796141321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8a40095f0 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b419b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b4197650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7ff8a800e9a0 tx=0x7ff8a800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b1ffb700 1 -- 172.21.15.90:0/3796141321 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a8009790 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b1ffb700 1 -- 172.21.15.90:0/3796141321 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a8010430 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.529+0000 7ff8b1ffb700 1 -- 172.21.15.90:0/3796141321 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a8018680 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.530+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8b419c640 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.530+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8b41a3e80 con 0x7ff8b4067270 2024-04-06T14:13:50.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8b1ffb700 1 -- 172.21.15.90:0/3796141321 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff8a80105a0 con 0x7ff8b4067270 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 Processor -- start 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 -- start start 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d6bb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d758a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d781d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0cc3990 con 0x5627b0d75de0 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0cc1350 con 0x5627b0d6c0a0 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0cbcc40 con 0x5627b0d6b780 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8b1ffb700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 0x7ff89c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.534+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d758a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d758a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57296/0 (socket says 172.21.15.90:57296) 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 0x7ff89c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.543 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/491122846 learned_addr learned my addr 172.21.15.90:0/491122846 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d781d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b1ffb700 1 -- 172.21.15.90:0/3796141321 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff8a8014070 con 0x7ff8b4067270 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d6bb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 -- 172.21.15.90:0/491122846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 msgr2=0x5627b0d758a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d758a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 -- 172.21.15.90:0/491122846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 msgr2=0x5627b0d781d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d781d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 -- 172.21.15.90:0/491122846 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5627b0b3e3b0 con 0x5627b0d6b780 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d6bb60 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4006010 tx=0x7ff8a400bdc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=904680a8c83a0b63 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 0x7ff89c0a37d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff8a40098e0 tx=0x7ff8a401f000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff89affd700 1 -- 172.21.15.90:0/491122846 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a400bb60 con 0x5627b0d6b780 2024-04-06T14:13:50.544 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff89affd700 1 -- 172.21.15.90:0/491122846 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a400be30 con 0x5627b0d6b780 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.535+0000 7ff89affd700 1 -- 172.21.15.90:0/491122846 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a40055d0 con 0x5627b0d6b780 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.536+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/491122846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 msgr2=0x5627b0d6bb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.536+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d6bb60 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4006010 tx=0x7ff8a400bdc0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.537+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/491122846 shutdown_connections 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.537+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d781d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.537+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d6bb60 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.537+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/491122846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d758a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.537+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/491122846 >> 172.21.15.90:0/491122846 conn(0x5627b0cca6a0 msgr2=0x5627b0d66a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.538+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/491122846 shutdown_connections 2024-04-06T14:13:50.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.538+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/491122846 wait complete. 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.538+0000 7ff8c6fe9b80 1 -- start start 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.538+0000 7ff8c6fe9b80 1 Processor -- start 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d75670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d72c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0c58d80 con 0x5627b0d75de0 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0c58a20 con 0x5627b0d6c0a0 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0b406a0 con 0x5627b0d6b780 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8c6fe9b80 10 client.0 _ll_register_callbacks cb 0x5627b0c62470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39948/0 (socket says 172.21.15.90:39948) 2024-04-06T14:13:50.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 -- 172.21.15.90:0/633766475 learned_addr learned my addr 172.21.15.90:0/633766475 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d72c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d75670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 -- 172.21.15.90:0/633766475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 msgr2=0x5627b0d75670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d75670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 -- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 msgr2=0x5627b0d72c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d72c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 -- 172.21.15.90:0/633766475 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8a4020040 con 0x5627b0d75de0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d72c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7ff8a400b690 tx=0x7ff8a4005820 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cfb9c37a9abd9679 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.539+0000 7ff893fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.540+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a4004a60 con 0x5627b0d75de0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.540+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff8a4004bc0 con 0x5627b0d75de0 2024-04-06T14:13:50.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.540+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5627b0c7d380 con 0x5627b0d75de0 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.540+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff8a4004d80 con 0x5627b0d75de0 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 10 client.16827 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5627b0aeab60 con 0x5627b0d75de0 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5627b0c7d380 con 0x5627b0d75de0 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff8a4052070 con 0x5627b0d75de0 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 10 client.16827 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 10 client.16827 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 20 client.16827 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0d7f560 0x5627b0d81980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 4 client.16827 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:50.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.541+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5627b0cb5700 con 0x5627b0d7f560 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.542+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0d7f560 0x5627b0d81980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.542+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0d7f560 0x5627b0d81980 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=295aa13fd4b6c892 server_cookie=7b6a68bda05d47c3 in_seq=0 out_seq=0 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.542+0000 7ff893fff700 10 client.16827 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff893fff700 1 -- 172.21.15.90:0/633766475 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5627b0cb5700 con 0x5627b0d7f560 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff893fff700 10 client.16827 handle_command_reply: tid=0 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 client.16827 shutdown 2024-04-06T14:13:50.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 msgr2=0x5627b0d731a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 secure :-1 s=READY pgs=670 cs=0 l=1 rev1=1 crypto rx=0x7ff8a400b690 tx=0x7ff8a4005820 comp rx=0 tx=0).stop 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 shutdown_connections 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d75de0 0x5627b0d731a0 unknown :-1 s=CLOSED pgs=670 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0d7f560 0x5627b0d81980 crc :-1 s=READY pgs=155 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d6b780 0x5627b0d75670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/633766475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d72c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 >> 172.21.15.90:0/633766475 conn(0x5627b0cca6a0 msgr2=0x5627b0d65c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.543+0000 7ff893fff700 0 client.16827 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.544+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 shutdown_connections 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.544+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/633766475 wait complete. 2024-04-06T14:13:50.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.544+0000 7ff8c6fe9b80 20 client.16827 trim_cache size 0 max 16384 2024-04-06T14:13:50.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.562+0000 7ff8c6fe9b80 1 Processor -- start 2024-04-06T14:13:50.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 -- start start 2024-04-06T14:13:50.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d75400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d71530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0d71a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0d741f0 con 0x5627b0d6b780 2024-04-06T14:13:50.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0d6ca40 con 0x5627b0d6c0a0 2024-04-06T14:13:50.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.564+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0d6da70 con 0x5627b0d75de0 2024-04-06T14:13:50.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.568+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d75400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.568+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d75400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39962/0 (socket says 172.21.15.90:39962) 2024-04-06T14:13:50.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.568+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/2900734238 learned_addr learned my addr 172.21.15.90:0/2900734238 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.570+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0d71a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d71530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b93d2700 1 -- 172.21.15.90:0/2900734238 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 msgr2=0x5627b0d71530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d71530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b93d2700 1 -- 172.21.15.90:0/2900734238 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 msgr2=0x5627b0d75400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d75400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.577 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.571+0000 7ff8b93d2700 1 -- 172.21.15.90:0/2900734238 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5627b0c82ea0 con 0x5627b0d75de0 2024-04-06T14:13:50.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.585+0000 7ff8b93d2700 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0d71a70 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4004670 tx=0x7ff8a4009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e54175e3c4433b8e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.600+0000 7ff89b7fe700 1 -- 172.21.15.90:0/2900734238 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a400bb60 con 0x5627b0d75de0 2024-04-06T14:13:50.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.600+0000 7ff89b7fe700 1 -- 172.21.15.90:0/2900734238 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a4009290 con 0x5627b0d75de0 2024-04-06T14:13:50.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.600+0000 7ff89b7fe700 1 -- 172.21.15.90:0/2900734238 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a4029470 con 0x5627b0d75de0 2024-04-06T14:13:50.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.602+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/2900734238 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 msgr2=0x5627b0d71a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.602+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0d71a70 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7ff8a4004670 tx=0x7ff8a4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.609+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/2900734238 shutdown_connections 2024-04-06T14:13:50.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.609+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0d71a70 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.609+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d75400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.609+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/2900734238 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d71530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.609+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/2900734238 >> 172.21.15.90:0/2900734238 conn(0x5627b0cca6a0 msgr2=0x5627b0d8a9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.615+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/2900734238 shutdown_connections 2024-04-06T14:13:50.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.617+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/2900734238 wait complete. 2024-04-06T14:13:50.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.617+0000 7ff8c6fe9b80 1 -- start start 2024-04-06T14:13:50.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.617+0000 7ff8c6fe9b80 1 Processor -- start 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d83700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0de9670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0de9bb0 con 0x5627b0d6b780 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0de9cf0 con 0x5627b0d6c0a0 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5627b0de9e30 con 0x5627b0d75de0 2024-04-06T14:13:50.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8c6fe9b80 10 client.0 _ll_register_callbacks cb 0x5627b0c62470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8b8bd1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39974/0 (socket says 172.21.15.90:39974) 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.619+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/417676245 learned_addr learned my addr 172.21.15.90:0/417676245 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/417676245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 msgr2=0x5627b0de9670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0de9670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 msgr2=0x5627b0d83700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d83700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 -- 172.21.15.90:0/417676245 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5627b0aeab60 con 0x5627b0d6b780 2024-04-06T14:13:50.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff8b8bd1700 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7ff8a8010c30 tx=0x7ff8a806a040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9aaff193bf7cfaff server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.620+0000 7ff893fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.621+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a806c040 con 0x5627b0d6b780 2024-04-06T14:13:50.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.621+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff8a809de70 con 0x5627b0d6b780 2024-04-06T14:13:50.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.624+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff8a400be30 con 0x5627b0d6b780 2024-04-06T14:13:50.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.624+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff8a80a24c0 con 0x5627b0d6b780 2024-04-06T14:13:50.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.624+0000 7ff8c6fe9b80 10 client.16833 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:50.624 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.624+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5627b0b42cd0 con 0x5627b0d6b780 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a400be30 con 0x5627b0d6b780 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff8a8096750 con 0x5627b0d6b780 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 10 client.16833 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 10 client.16833 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 20 client.16833 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:50.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0deb5d0 0x5627b0ded9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:50.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 4 client.16833 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:13:50.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.625+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5627b0b0bcf0 con 0x5627b0deb5d0 2024-04-06T14:13:50.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.626+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0deb5d0 0x5627b0ded9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff893fff700 10 client.16833 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff8b3fff700 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0deb5d0 0x5627b0ded9f0 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ff6aad156b6be0d8 server_cookie=ad83c48e15ae3a38 in_seq=0 out_seq=0 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff893fff700 1 -- 172.21.15.90:0/417676245 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5627b0b0bcf0 con 0x5627b0deb5d0 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff893fff700 10 client.16833 handle_command_reply: tid=0 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff8c6fe9b80 1 client.16833 shutdown 2024-04-06T14:13:50.634 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 msgr2=0x5627b0d831c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.633+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 secure :-1 s=READY pgs=671 cs=0 l=1 rev1=1 crypto rx=0x7ff8a8010c30 tx=0x7ff8a806a040 comp rx=0 tx=0).stop 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 shutdown_connections 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5627b0deb5d0 0x5627b0ded9f0 crc :-1 s=READY pgs=156 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5627b0d75de0 0x5627b0de9670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5627b0d6b780 0x5627b0d831c0 unknown :-1 s=CLOSED pgs=671 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 --2- 172.21.15.90:0/417676245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5627b0d6c0a0 0x5627b0d83700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 >> 172.21.15.90:0/417676245 conn(0x5627b0cca6a0 msgr2=0x5627b0d72ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff893fff700 0 client.16833 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:50.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 shutdown_connections 2024-04-06T14:13:50.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.634+0000 7ff8c6fe9b80 1 -- 172.21.15.90:0/417676245 wait complete. 2024-04-06T14:13:50.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.635+0000 7ff8c6fe9b80 20 client.16833 trim_cache size 0 max 16384 2024-04-06T14:13:50.637 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 msgr2=0x7ff89c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 0x7ff89c0a37d0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ff8a40098e0 tx=0x7ff8a401f000 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 msgr2=0x7ff8b4197110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 secure :-1 s=READY pgs=669 cs=0 l=1 rev1=1 crypto rx=0x7ff8a800e9a0 tx=0x7ff8a800ecb0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 shutdown_connections 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff89c0a1310 0x7ff89c0a37d0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff8b4103f40 0x7ff8b419b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff8b4067270 0x7ff8b4197110 unknown :-1 s=CLOSED pgs=669 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 --2- 172.21.15.90:0/3796141321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff8b40ff1d0 0x7ff8b4197650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 >> 172.21.15.90:0/3796141321 conn(0x7ff8b4074b50 msgr2=0x7ff8b40fe1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 shutdown_connections 2024-04-06T14:13:50.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:50.636+0000 7ff8bae35700 1 -- 172.21.15.90:0/3796141321 wait complete. 2024-04-06T14:13:51.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:50 smithi090 ceph-mon[43420]: pgmap v260: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:51.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:50 smithi134 ceph-mon[47549]: pgmap v260: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:51.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:50 smithi105 ceph-mon[46969]: pgmap v260: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 1.15k op/s 2024-04-06T14:13:51.654 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:13:51.654 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:13:51.654 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:13:51.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 -- 172.21.15.90:0/766242181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c10c6b0 msgr2=0x7f733c114230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 --2- 172.21.15.90:0/766242181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c10c6b0 0x7f733c114230 secure :-1 s=READY pgs=672 cs=0 l=1 rev1=1 crypto rx=0x7f7334009f90 tx=0x7f733400b330 comp rx=0 tx=0).stop 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 -- 172.21.15.90:0/766242181 shutdown_connections 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 --2- 172.21.15.90:0/766242181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c10c6b0 0x7f733c114230 unknown :-1 s=CLOSED pgs=672 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 --2- 172.21.15.90:0/766242181 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f733c107590 0x7f733c107970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 --2- 172.21.15.90:0/766242181 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c107f40 0x7f733c10bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 -- 172.21.15.90:0/766242181 >> 172.21.15.90:0/766242181 conn(0x7f733c0752a0 msgr2=0x7f733c0776c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 -- 172.21.15.90:0/766242181 shutdown_connections 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.825+0000 7f7342388700 1 -- 172.21.15.90:0/766242181 wait complete. 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.826+0000 7f7342388700 1 Processor -- start 2024-04-06T14:13:51.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.826+0000 7f7342388700 1 -- start start 2024-04-06T14:13:51.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.826+0000 7f7342388700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f733c107590 0x7f733c1a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.826+0000 7f7342388700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.826+0000 7f7342388700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7342388700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f733c19e260 con 0x7f733c19d7d0 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7342388700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f733c19e3d0 con 0x7f733c10c6b0 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7342388700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f733c19e590 con 0x7f733c107590 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f733b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39998/0 (socket says 172.21.15.90:39998) 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f733b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57366/0 (socket says 172.21.15.90:57366) 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.827+0000 7f7340925700 1 -- 172.21.15.90:0/592497947 learned_addr learned my addr 172.21.15.90:0/592497947 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 -- 172.21.15.90:0/592497947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f733c107590 msgr2=0x7f733c1a3680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f733c107590 0x7f733c1a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 -- 172.21.15.90:0/592497947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 msgr2=0x7f733c1a3bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 -- 172.21.15.90:0/592497947 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73300095f0 con 0x7f733c19d7d0 2024-04-06T14:13:51.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.828+0000 7f7340925700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f7334006070 tx=0x7f7334008cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f73397fa700 1 -- 172.21.15.90:0/592497947 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f733400e040 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f733b7fe700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f73397fa700 1 -- 172.21.15.90:0/592497947 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f733400bda0 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f73397fa700 1 -- 172.21.15.90:0/592497947 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7334005280 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7334009c40 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.829+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f733c1aa770 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.831+0000 7f73397fa700 1 -- 172.21.15.90:0/592497947 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f733401d470 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.832+0000 7f73397fa700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 0x7f73240a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.832+0000 7f73397fa700 1 -- 172.21.15.90:0/592497947 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f73340ce0f0 con 0x7f733c19d7d0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.833+0000 7f733bfff700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 0x7f73240a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.833+0000 7f734e53cb80 1 Processor -- start 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.833+0000 7f734e53cb80 1 -- start start 2024-04-06T14:13:51.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.833+0000 7f733bfff700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 0x7f73240a38a0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f732c005950 tx=0x7f732c0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.833+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864475800 0x55d864475be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864476120 0x55d86447f920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f734e53cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d8643cda10 con 0x55d864475800 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f734e53cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d8643cb3d0 con 0x55d86447fe60 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f734e53cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d8643c6cc0 con 0x55d864476120 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57378/0 (socket says 172.21.15.90:57378) 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 -- 172.21.15.90:0/4251305007 learned_addr learned my addr 172.21.15.90:0/4251305007 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f733bfff700 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864475800 0x55d864475be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 -- 172.21.15.90:0/4251305007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864476120 msgr2=0x55d86447f920 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864476120 0x55d86447f920 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 -- 172.21.15.90:0/4251305007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864475800 msgr2=0x55d864475be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864475800 0x55d864475be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.837 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 -- 172.21.15.90:0/4251305007 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d864248430 con 0x55d86447fe60 2024-04-06T14:13:51.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.834+0000 7f7340925700 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f7334097ce0 tx=0x7f7334008fc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7ddb68d6a8141c7e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.835+0000 7f73227fc700 1 -- 172.21.15.90:0/4251305007 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f733409b970 con 0x55d86447fe60 2024-04-06T14:13:51.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.835+0000 7f73227fc700 1 -- 172.21.15.90:0/4251305007 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f733409b270 con 0x55d86447fe60 2024-04-06T14:13:51.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.835+0000 7f73227fc700 1 -- 172.21.15.90:0/4251305007 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7334097b60 con 0x55d86447fe60 2024-04-06T14:13:51.839 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- 172.21.15.90:0/4251305007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 msgr2=0x55d864482250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f7334097ce0 tx=0x7f7334008fc0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- 172.21.15.90:0/4251305007 shutdown_connections 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d86447fe60 0x55d864482250 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864475800 0x55d864475be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 --2- 172.21.15.90:0/4251305007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864476120 0x55d86447f920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- 172.21.15.90:0/4251305007 >> 172.21.15.90:0/4251305007 conn(0x55d8643d4720 msgr2=0x55d864470b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- 172.21.15.90:0/4251305007 shutdown_connections 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- 172.21.15.90:0/4251305007 wait complete. 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 -- start start 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.836+0000 7f734e53cb80 1 Processor -- start 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d86447f890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d86447d2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d864362e00 con 0x55d864476120 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d864362aa0 con 0x55d864475800 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d86424a720 con 0x55d86447fe60 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f734e53cb80 10 client.0 _ll_register_callbacks cb 0x55d86436c4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40030/0 (socket says 172.21.15.90:40030) 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 -- 172.21.15.90:0/3961771861 learned_addr learned my addr 172.21.15.90:0/3961771861 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733bfff700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d86447f890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f7340925700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d86447d2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 -- 172.21.15.90:0/3961771861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 msgr2=0x55d86447d2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d86447d2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 -- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 msgr2=0x55d86447f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:51.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d86447f890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f733b7fe700 1 -- 172.21.15.90:0/3961771861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f733c1aa770 con 0x55d864476120 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.837+0000 7f7340925700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d86447d2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.838+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f7330010770 tx=0x7f7330010a80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7bded6e4d68f71b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.838+0000 7f731b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.838+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7330004a30 con 0x55d864476120 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.838+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7330016070 con 0x55d864476120 2024-04-06T14:13:51.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.839+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d864387400 con 0x55d864476120 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.840+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7330004b90 con 0x55d864476120 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.841+0000 7f734e53cb80 10 client.16845 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.841+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d8641f4be0 con 0x55d864476120 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.841+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d864387400 con 0x55d864476120 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7330003680 con 0x55d864476120 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 10 client.16845 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 10 client.16845 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 20 client.16845 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644a0830 0x55d8644a2c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 4 client.16845 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d8643bf780 con 0x55d8644a0830 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f733bfff700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644a0830 0x55d8644a2c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f731b7fe700 10 client.16845 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.842+0000 7f733bfff700 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644a0830 0x55d8644a2c50 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=730fe90ce6dcc307 server_cookie=12d195c5b039b08b in_seq=0 out_seq=0 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f731b7fe700 1 -- 172.21.15.90:0/3961771861 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d8643bf780 con 0x55d8644a0830 2024-04-06T14:13:51.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f731b7fe700 10 client.16845 handle_command_reply: tid=0 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 client.16845 shutdown 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 msgr2=0x55d86447cb80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 secure :-1 s=READY pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f7330010770 tx=0x7f7330010a80 comp rx=0 tx=0).stop 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 shutdown_connections 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644a0830 0x55d8644a2c50 crc :-1 s=READY pgs=157 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d86447d2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d86447f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3961771861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864476120 0x55d86447cb80 secure :-1 s=CLOSED pgs=674 cs=0 l=1 rev1=1 crypto rx=0x7f7330010770 tx=0x7f7330010a80 comp rx=0 tx=0).stop 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 >> 172.21.15.90:0/3961771861 conn(0x55d8643d4720 msgr2=0x55d86446fca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.843+0000 7f731b7fe700 0 client.16845 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:51.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.844+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 shutdown_connections 2024-04-06T14:13:51.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.844+0000 7f734e53cb80 1 -- 172.21.15.90:0/3961771861 wait complete. 2024-04-06T14:13:51.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.844+0000 7f734e53cb80 20 client.16845 trim_cache size 0 max 16384 2024-04-06T14:13:51.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 Processor -- start 2024-04-06T14:13:51.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 -- start start 2024-04-06T14:13:51.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d864497900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d864477ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d86449dfe0 con 0x55d864478400 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d86447d680 con 0x55d864475800 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.866+0000 7f734e53cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d86449d3d0 con 0x55d86447fe60 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40034/0 (socket says 172.21.15.90:40034) 2024-04-06T14:13:51.868 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 -- 172.21.15.90:0/391835990 learned_addr learned my addr 172.21.15.90:0/391835990 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 -- 172.21.15.90:0/391835990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 msgr2=0x55d864477ec0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d864477ec0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 -- 172.21.15.90:0/391835990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 msgr2=0x55d864497900 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d864497900 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.867+0000 7f7340925700 1 -- 172.21.15.90:0/391835990 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d86446cbe0 con 0x55d864478400 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.868+0000 7f7340925700 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7f73340cbe20 tx=0x7f733409b0f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4761cc1a2db79d55 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.868+0000 7f7322ffd700 1 -- 172.21.15.90:0/391835990 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73340a3660 con 0x55d864478400 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.868+0000 7f7322ffd700 1 -- 172.21.15.90:0/391835990 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f733409f0a0 con 0x55d864478400 2024-04-06T14:13:51.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.868+0000 7f7322ffd700 1 -- 172.21.15.90:0/391835990 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f733409b4d0 con 0x55d864478400 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 -- 172.21.15.90:0/391835990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 msgr2=0x55d86447b470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 secure :-1 s=READY pgs=675 cs=0 l=1 rev1=1 crypto rx=0x7f73340cbe20 tx=0x7f733409b0f0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 -- 172.21.15.90:0/391835990 shutdown_connections 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d864478400 0x55d86447b470 unknown :-1 s=CLOSED pgs=675 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d864497900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 --2- 172.21.15.90:0/391835990 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d86447fe60 0x55d864477ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 -- 172.21.15.90:0/391835990 >> 172.21.15.90:0/391835990 conn(0x55d8643d4720 msgr2=0x55d86447c0e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 -- 172.21.15.90:0/391835990 shutdown_connections 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.869+0000 7f734e53cb80 1 -- 172.21.15.90:0/391835990 wait complete. 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 -- start start 2024-04-06T14:13:51.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 Processor -- start 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d8644eec60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864478400 0x55d8644ef1a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 0x55d864496af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d864497110 con 0x55d86447fe60 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d864497280 con 0x55d864475800 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d864497440 con 0x55d864478400 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f734e53cb80 10 client.0 _ll_register_callbacks cb 0x55d86436c4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 0x55d864496af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f733bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d8644eec60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f7340925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 0x55d864496af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40038/0 (socket says 172.21.15.90:40038) 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f7340925700 1 -- 172.21.15.90:0/3636834869 learned_addr learned my addr 172.21.15.90:0/3636834869 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.870+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864478400 0x55d8644ef1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 -- 172.21.15.90:0/3636834869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864478400 msgr2=0x55d8644ef1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:51.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864478400 0x55d8644ef1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 -- 172.21.15.90:0/3636834869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 msgr2=0x55d864496af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 0x55d864496af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 -- 172.21.15.90:0/3636834869 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d864248430 con 0x55d864475800 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f733bfff700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d8644eec60 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x55d8644a0c10 tx=0x7f732c005d20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3273eb7fe7054a0d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f731b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f732c024040 con 0x55d864475800 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f732c017040 con 0x55d864475800 2024-04-06T14:13:51.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.871+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f733409b270 con 0x55d864475800 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.874+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f732c0264c0 con 0x55d864475800 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.874+0000 7f734e53cb80 10 client.26558 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.874+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d86438cf20 con 0x55d864475800 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f733409b270 con 0x55d864475800 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f732c026cf0 con 0x55d864475800 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 10 client.26558 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 10 client.26558 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 20 client.26558 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644f6be0 0x55d8644f9000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 4 client.26558 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55d864215d70 con 0x55d8644f6be0 2024-04-06T14:13:51.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.875+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644f6be0 0x55d8644f9000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:51.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.876+0000 7f733b7fe700 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644f6be0 0x55d8644f9000 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e6e39b373655251 server_cookie=230b2e3759e22824 in_seq=0 out_seq=0 2024-04-06T14:13:51.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.876+0000 7f731b7fe700 10 client.26558 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:51.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.877+0000 7f731b7fe700 1 -- 172.21.15.90:0/3636834869 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55d864215d70 con 0x55d8644f6be0 2024-04-06T14:13:51.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.877+0000 7f731b7fe700 10 client.26558 handle_command_reply: tid=0 2024-04-06T14:13:51.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.877+0000 7f734e53cb80 1 client.26558 shutdown 2024-04-06T14:13:51.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.877+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 msgr2=0x55d8644eec60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.877+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d8644eec60 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x55d8644a0c10 tx=0x7f732c005d20 comp rx=0 tx=0).stop 2024-04-06T14:13:51.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 shutdown_connections 2024-04-06T14:13:51.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d86447fe60 0x55d864496af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d864475800 0x55d8644eec60 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d8644f6be0 0x55d8644f9000 crc :-1 s=READY pgs=158 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 --2- 172.21.15.90:0/3636834869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d864478400 0x55d8644ef1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 >> 172.21.15.90:0/3636834869 conn(0x55d8643d4720 msgr2=0x55d8644747f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f731b7fe700 0 client.26558 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:51.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 shutdown_connections 2024-04-06T14:13:51.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.879+0000 7f734e53cb80 1 -- 172.21.15.90:0/3636834869 wait complete. 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.880+0000 7f734e53cb80 20 client.26558 trim_cache size 0 max 16384 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "3012adf8-dd06-4194-bab2-903717236497", 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:13:51.881 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:51.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 msgr2=0x7f73240a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 0x7f73240a38a0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f732c005950 tx=0x7f732c0058e0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 msgr2=0x7f733c19dc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 secure :-1 s=READY pgs=673 cs=0 l=1 rev1=1 crypto rx=0x7f7334006070 tx=0x7f7334008cf0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 shutdown_connections 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f73240a13e0 0x7f73240a38a0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f733c19d7d0 0x7f733c19dc50 unknown :-1 s=CLOSED pgs=673 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f733c107590 0x7f733c1a3680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 --2- 172.21.15.90:0/592497947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f733c10c6b0 0x7f733c1a3bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 >> 172.21.15.90:0/592497947 conn(0x7f733c0752a0 msgr2=0x7f733c077590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 shutdown_connections 2024-04-06T14:13:51.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:51.881+0000 7f7342388700 1 -- 172.21.15.90:0/592497947 wait complete. 2024-04-06T14:13:51.898 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:13:51.898 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 -- 172.21.15.90:0/3949423852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 msgr2=0x7f62f010cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/3949423852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f010cb00 secure :-1 s=READY pgs=676 cs=0 l=1 rev1=1 crypto rx=0x7f62e800b0a0 tx=0x7f62e800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 -- 172.21.15.90:0/3949423852 shutdown_connections 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/3949423852 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f010cb00 unknown :-1 s=CLOSED pgs=676 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/3949423852 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62f00ffe60 0x7f62f0100240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.088 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/3949423852 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f0100810 0x7f62f0104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 -- 172.21.15.90:0/3949423852 >> 172.21.15.90:0/3949423852 conn(0x7f62f0076db0 msgr2=0x7f62f00771c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 -- 172.21.15.90:0/3949423852 shutdown_connections 2024-04-06T14:13:52.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.087+0000 7f62f5ceb700 1 -- 172.21.15.90:0/3949423852 wait complete. 2024-04-06T14:13:52.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.088+0000 7f62f5ceb700 1 Processor -- start 2024-04-06T14:13:52.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.088+0000 7f62f5ceb700 1 -- start start 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f01955a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62f0199940 0x7f62f0199dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62f019a3c0 con 0x7f62f0104f80 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62f019a530 con 0x7f62f00ffe60 2024-04-06T14:13:52.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62f5ceb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62f019a6f0 con 0x7f62f0199940 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57410/0 (socket says 172.21.15.90:57410) 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 -- 172.21.15.90:0/2475593248 learned_addr learned my addr 172.21.15.90:0/2475593248 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f01955a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 -- 172.21.15.90:0/2475593248 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62f0199940 msgr2=0x7f62f0199dc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62f0199940 0x7f62f0199dc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 -- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 msgr2=0x7f62f01955a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f01955a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.090+0000 7f62ef7fe700 1 -- 172.21.15.90:0/2475593248 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62e8009d00 con 0x7f62f00ffe60 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f62e000e9a0 tx=0x7f62e000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62ed7fa700 1 -- 172.21.15.90:0/2475593248 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e0009790 con 0x7f62f00ffe60 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62ed7fa700 1 -- 172.21.15.90:0/2475593248 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f62e000cbd0 con 0x7f62f00ffe60 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62ed7fa700 1 -- 172.21.15.90:0/2475593248 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e0018680 con 0x7f62f00ffe60 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62f01a1c50 con 0x7f62f00ffe60 2024-04-06T14:13:52.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.091+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62f01a20e0 con 0x7f62f00ffe60 2024-04-06T14:13:52.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.095+0000 7f6301e9fb80 1 Processor -- start 2024-04-06T14:13:52.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.095+0000 7f6301e9fb80 1 -- start start 2024-04-06T14:13:52.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.095+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9733a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d964c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d8bc990 con 0x55e29d972fc0 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d8ba350 con 0x55e29d965140 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d8b5c40 con 0x55e29d964780 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57422/0 (socket says 172.21.15.90:57422) 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 -- 172.21.15.90:0/742233871 learned_addr learned my addr 172.21.15.90:0/742233871 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62effff700 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9733a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 -- 172.21.15.90:0/742233871 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 msgr2=0x55e29d964c00 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d964c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 -- 172.21.15.90:0/742233871 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 msgr2=0x55e29d9733a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9733a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.096+0000 7f62e7fff700 1 -- 172.21.15.90:0/742233871 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e29d7373b0 con 0x55e29d965140 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.097+0000 7f62ed7fa700 1 -- 172.21.15.90:0/2475593248 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f62e000cd40 con 0x7f62f00ffe60 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.097+0000 7f62ed7fa700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 0x7f62dc0a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62ed7fa700 1 -- 172.21.15.90:0/2475593248 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f62e0014070 con 0x7f62f00ffe60 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62e7fff700 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f62d800cdf0 tx=0x7f62d800c590 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ef36937f9344a8dd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62e5ffb700 1 -- 172.21.15.90:0/742233871 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d8011040 con 0x55e29d965140 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62e5ffb700 1 -- 172.21.15.90:0/742233871 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f62d800c2e0 con 0x55e29d965140 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62e5ffb700 1 -- 172.21.15.90:0/742233871 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d8003c20 con 0x55e29d965140 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.098+0000 7f62effff700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 0x7f62dc0a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 -- 172.21.15.90:0/742233871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 msgr2=0x55e29d979d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f62d800cdf0 tx=0x7f62d800c590 comp rx=0 tx=0).stop 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 -- 172.21.15.90:0/742233871 shutdown_connections 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d979d00 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9733a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/742233871 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d964c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 -- 172.21.15.90:0/742233871 >> 172.21.15.90:0/742233871 conn(0x55e29d8c36a0 msgr2=0x55e29d95f720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f62effff700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 0x7f62dc0a3850 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f62e8009fd0 tx=0x7f62e8006210 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 -- 172.21.15.90:0/742233871 shutdown_connections 2024-04-06T14:13:52.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.099+0000 7f6301e9fb80 1 -- 172.21.15.90:0/742233871 wait complete. 2024-04-06T14:13:52.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 -- start start 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 Processor -- start 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d979c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d851d80 con 0x55e29d972fc0 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d851a20 con 0x55e29d965140 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d7396a0 con 0x55e29d964780 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.100+0000 7f6301e9fb80 10 client.0 _ll_register_callbacks cb 0x55e29d85b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:52.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62effff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57426/0 (socket says 172.21.15.90:57426) 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62effff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40088/0 (socket says 172.21.15.90:40088) 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 -- 172.21.15.90:0/2515788373 learned_addr learned my addr 172.21.15.90:0/2515788373 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 -- 172.21.15.90:0/2515788373 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 msgr2=0x55e29d979c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d979c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 -- 172.21.15.90:0/2515788373 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 msgr2=0x55e29d9777c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 -- 172.21.15.90:0/2515788373 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62d800caa0 con 0x55e29d965140 2024-04-06T14:13:52.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62effff700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:13:52.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f62d8000c00 tx=0x7f62d80156e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=51c14198866f5a0a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62ceffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:52.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d8011070 con 0x55e29d965140 2024-04-06T14:13:52.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f62d8005df0 con 0x55e29d965140 2024-04-06T14:13:52.107 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e29d8ae700 con 0x55e29d965140 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.101+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f62d8015e10 con 0x55e29d965140 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.102+0000 7f6301e9fb80 10 client.26567 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.102+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e29d6e3b60 con 0x55e29d965140 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e29d8ae700 con 0x55e29d965140 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f62d8009240 con 0x55e29d965140 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 10 client.26567 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 10 client.26567 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:52.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 20 client.26567 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:52.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d969950 0x55e29d96bd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 4 client.26567 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:13:52.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e29d704cf0 con 0x55e29d969950 2024-04-06T14:13:52.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.103+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d969950 0x55e29d96bd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.104+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d969950 0x55e29d96bd70 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ee34f31073b37abf server_cookie=dc0a7969f72bc8d6 in_seq=0 out_seq=0 2024-04-06T14:13:52.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.104+0000 7f62ceffd700 10 client.26567 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:52.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.104+0000 7f62ceffd700 1 -- 172.21.15.90:0/2515788373 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55e29d704cf0 con 0x55e29d969950 2024-04-06T14:13:52.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f62ceffd700 10 client.26567 handle_command_reply: tid=0 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 client.26567 shutdown 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 msgr2=0x55e29d977080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f62d8000c00 tx=0x7f62d80156e0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 shutdown_connections 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d9777c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d969950 0x55e29d96bd70 crc :-1 s=READY pgs=159 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d964780 0x55e29d979c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2515788373 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d977080 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 >> 172.21.15.90:0/2515788373 conn(0x55e29d8c36a0 msgr2=0x55e29d95f320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f62ceffd700 0 client.26567 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 shutdown_connections 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2515788373 wait complete. 2024-04-06T14:13:52.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.105+0000 7f6301e9fb80 20 client.26567 trim_cache size 0 max 16384 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.125+0000 7f6301e9fb80 1 Processor -- start 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 -- start start 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d964780 0x55e29d965ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d965140 0x55e29d974b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d95bdd0 con 0x55e29d972fc0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d964b60 con 0x55e29d964780 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.126+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d964cd0 con 0x55e29d965140 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40104/0 (socket says 172.21.15.90:40104) 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 -- 172.21.15.90:0/2014607273 learned_addr learned my addr 172.21.15.90:0/2014607273 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d964780 0x55e29d965ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d965140 0x55e29d974b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 -- 172.21.15.90:0/2014607273 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d965140 msgr2=0x55e29d974b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d965140 0x55e29d974b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 -- 172.21.15.90:0/2014607273 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d964780 msgr2=0x55e29d965ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d964780 0x55e29d965ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 -- 172.21.15.90:0/2014607273 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e29d87bea0 con 0x55e29d972fc0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62effff700 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f62e8007eb0 tx=0x7f62e801c000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=264db540d63710bb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62e67fc700 1 -- 172.21.15.90:0/2014607273 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e8023040 con 0x55e29d972fc0 2024-04-06T14:13:52.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62e67fc700 1 -- 172.21.15.90:0/2014607273 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f62e802a910 con 0x55e29d972fc0 2024-04-06T14:13:52.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.127+0000 7f62e67fc700 1 -- 172.21.15.90:0/2014607273 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62e800e040 con 0x55e29d972fc0 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2014607273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 msgr2=0x55e29d95b890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 secure :-1 s=READY pgs=677 cs=0 l=1 rev1=1 crypto rx=0x7f62e8007eb0 tx=0x7f62e801c000 comp rx=0 tx=0).stop 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2014607273 shutdown_connections 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d972fc0 0x55e29d95b890 unknown :-1 s=CLOSED pgs=677 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d964780 0x55e29d965ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2014607273 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d965140 0x55e29d974b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2014607273 >> 172.21.15.90:0/2014607273 conn(0x55e29d8c36a0 msgr2=0x55e29d9983a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2014607273 shutdown_connections 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.128+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2014607273 wait complete. 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.129+0000 7f6301e9fb80 1 -- start start 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.129+0000 7f6301e9fb80 1 Processor -- start 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d964780 0x55e29d97e300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d972fc0 0x55e29d96cd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d96d2b0 con 0x55e29d964780 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d96d420 con 0x55e29d965140 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e29d96d5e0 con 0x55e29d972fc0 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f6301e9fb80 10 client.0 _ll_register_callbacks cb 0x55e29d85b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:57438/0 (socket says 172.21.15.90:57438) 2024-04-06T14:13:52.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 -- 172.21.15.90:0/2218667907 learned_addr learned my addr 172.21.15.90:0/2218667907 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 -- 172.21.15.90:0/2218667907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d972fc0 msgr2=0x55e29d96cd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d972fc0 0x55e29d96cd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 -- 172.21.15.90:0/2218667907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d964780 msgr2=0x55e29d97e300 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d964780 0x55e29d97e300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.130+0000 7f62e7fff700 1 -- 172.21.15.90:0/2218667907 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62f010d370 con 0x55e29d965140 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f62e7fff700 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f62d800c840 tx=0x7f62d8009ea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f25e78d267266f18 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f62ceffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f62d8011040 con 0x55e29d965140 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f62d8004fe0 con 0x55e29d965140 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e29d876380 con 0x55e29d965140 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f62d80051e0 con 0x55e29d965140 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f6301e9fb80 10 client.26573 fetch_fsmap learned FSMap version 24 2024-04-06T14:13:52.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.131+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e29d73bcd0 con 0x55e29d965140 2024-04-06T14:13:52.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e29d876380 con 0x55e29d965140 2024-04-06T14:13:52.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f62d8003680 con 0x55e29d965140 2024-04-06T14:13:52.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 10 client.26573 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:13:52.136 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 10 client.26573 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 20 client.26573 populate_metadata read hostname 'smithi090' 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d9f55b0 0x55e29d9f79d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 4 client.26573 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.132+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55e29d739ce0 con 0x55e29d9f55b0 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d9f55b0 0x55e29d9f79d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f62ceffd700 10 client.26573 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f62ef7fe700 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d9f55b0 0x55e29d9f79d0 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6ec5d4ed4163541 server_cookie=63141e456f4fbcd1 in_seq=0 out_seq=0 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f62ceffd700 1 -- 172.21.15.90:0/2218667907 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55e29d739ce0 con 0x55e29d9f55b0 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f62ceffd700 10 client.26573 handle_command_reply: tid=0 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.133+0000 7f6301e9fb80 1 client.26573 shutdown 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 msgr2=0x55e29d96c7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f62d800c840 tx=0x7f62d8009ea0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 shutdown_connections 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e29d9f55b0 0x55e29d9f79d0 crc :-1 s=READY pgs=160 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e29d972fc0 0x55e29d96cd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e29d964780 0x55e29d97e300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 --2- 172.21.15.90:0/2218667907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e29d965140 0x55e29d96c7f0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 >> 172.21.15.90:0/2218667907 conn(0x55e29d8c36a0 msgr2=0x55e29d96b960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f62ceffd700 0 client.26573 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 shutdown_connections 2024-04-06T14:13:52.138 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 1 -- 172.21.15.90:0/2218667907 wait complete. 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.134+0000 7f6301e9fb80 20 client.26573 trim_cache size 0 max 16384 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "3012adf8-dd06-4194-bab2-903717236497": { 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "3012adf8-dd06-4194-bab2-903717236497", 2024-04-06T14:13:52.139 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:13:52.140 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:13:52.140 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:13:52.140 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:13:52.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 msgr2=0x7f62dc0a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 0x7f62dc0a3850 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7f62e8009fd0 tx=0x7f62e8006210 comp rx=0 tx=0).stop 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 msgr2=0x7f62f0195060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f62e000e9a0 tx=0x7f62e000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 shutdown_connections 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f62dc0a1390 0x7f62dc0a3850 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62f0199940 0x7f62f0199dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62f00ffe60 0x7f62f0195060 secure :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f62e000e9a0 tx=0x7f62e000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 --2- 172.21.15.90:0/2475593248 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f62f0104f80 0x7f62f01955a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:13:52.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 >> 172.21.15.90:0/2475593248 conn(0x7f62f0076db0 msgr2=0x7f62f00fdbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:13:52.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 shutdown_connections 2024-04-06T14:13:52.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:13:52.137+0000 7f62f5ceb700 1 -- 172.21.15.90:0/2475593248 wait complete. 2024-04-06T14:13:52.154 INFO:tasks.cephfs.filesystem:scrub status for tag:3012adf8-dd06-4194-bab2-903717236497 - {'path': '/', 'tag': '3012adf8-dd06-4194-bab2-903717236497', 'options': 'recursive,force'} 2024-04-06T14:13:53.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 485.0 s, 1894.9 tps, lat 52.771 ms stddev 61.090 2024-04-06T14:13:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:52 smithi134 ceph-mon[47549]: pgmap v261: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 21 MiB/s wr, 1.41k op/s 2024-04-06T14:13:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:52 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:13:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:52 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:52 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:13:53.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:52 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:53.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:52 smithi105 ceph-mon[46969]: pgmap v261: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 21 MiB/s wr, 1.41k op/s 2024-04-06T14:13:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:52 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:13:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:52 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:52 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:13:53.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:52 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:52 smithi090 ceph-mon[43420]: pgmap v261: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 21 MiB/s wr, 1.41k op/s 2024-04-06T14:13:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:52 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:13:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:52 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:52 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:13:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:52 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:13:55.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:54 smithi134 ceph-mon[47549]: pgmap v262: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 11 MiB/s wr, 656 op/s 2024-04-06T14:13:55.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:54 smithi105 ceph-mon[46969]: pgmap v262: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 11 MiB/s wr, 656 op/s 2024-04-06T14:13:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:54 smithi090 ceph-mon[43420]: pgmap v262: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 95 KiB/s rd, 11 MiB/s wr, 656 op/s 2024-04-06T14:13:56.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:55 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:13:56.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:55 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:13:56.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:55 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:13:57.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:56 smithi134 ceph-mon[47549]: pgmap v263: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 16 MiB/s wr, 933 op/s 2024-04-06T14:13:57.252 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:56 smithi105 ceph-mon[46969]: pgmap v263: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 16 MiB/s wr, 933 op/s 2024-04-06T14:13:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:56 smithi090 ceph-mon[43420]: pgmap v263: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 16 MiB/s wr, 933 op/s 2024-04-06T14:13:58.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 490.0 s, 1874.4 tps, lat 53.707 ms stddev 64.330 2024-04-06T14:13:58.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:13:58 smithi105 ceph-mon[46969]: pgmap v264: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:13:59.042 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:13:58 smithi090 ceph-mon[43420]: pgmap v264: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:13:59.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:13:58 smithi134 ceph-mon[47549]: pgmap v264: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:14:01.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:00 smithi134 ceph-mon[47549]: pgmap v265: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:14:01.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:00 smithi105 ceph-mon[46969]: pgmap v265: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:14:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:00 smithi090 ceph-mon[43420]: pgmap v265: 97 pgs: 97 active+clean; 7.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 9.3 MiB/s wr, 552 op/s 2024-04-06T14:14:03.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 495.0 s, 763.2 tps, lat 117.461 ms stddev 134.191 2024-04-06T14:14:03.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:02 smithi105 ceph-mon[46969]: pgmap v266: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 15 MiB/s wr, 893 op/s 2024-04-06T14:14:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:02 smithi090 ceph-mon[43420]: pgmap v266: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 15 MiB/s wr, 893 op/s 2024-04-06T14:14:03.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:02 smithi134 ceph-mon[47549]: pgmap v266: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 15 MiB/s wr, 893 op/s 2024-04-06T14:14:04.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:04 smithi090 ceph-mon[43420]: pgmap v267: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 10 MiB/s wr, 624 op/s 2024-04-06T14:14:04.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:04 smithi134 ceph-mon[47549]: pgmap v267: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 10 MiB/s wr, 624 op/s 2024-04-06T14:14:04.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:04 smithi105 ceph-mon[46969]: pgmap v267: 97 pgs: 97 active+clean; 7.9 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 10 MiB/s wr, 624 op/s 2024-04-06T14:14:07.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:06 smithi134 ceph-mon[47549]: pgmap v268: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 28 MiB/s wr, 2.04k op/s 2024-04-06T14:14:07.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:06 smithi105 ceph-mon[46969]: pgmap v268: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 28 MiB/s wr, 2.04k op/s 2024-04-06T14:14:07.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:06 smithi090 ceph-mon[43420]: pgmap v268: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 28 MiB/s wr, 2.04k op/s 2024-04-06T14:14:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 500.0 s, 1070.6 tps, lat 102.059 ms stddev 158.138 2024-04-06T14:14:09.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:08 smithi090 ceph-mon[43420]: pgmap v269: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:09.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:08 smithi134 ceph-mon[47549]: pgmap v269: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:09.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:08 smithi105 ceph-mon[46969]: pgmap v269: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:11.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:10 smithi134 ceph-mon[47549]: pgmap v270: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:11.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:10 smithi105 ceph-mon[46969]: pgmap v270: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:11.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:10 smithi090 ceph-mon[43420]: pgmap v270: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 24 MiB/s wr, 1.76k op/s 2024-04-06T14:14:12.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:11 smithi134 ceph-mon[47549]: osdmap e89: 12 total, 12 up, 12 in 2024-04-06T14:14:12.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:11 smithi105 ceph-mon[46969]: osdmap e89: 12 total, 12 up, 12 in 2024-04-06T14:14:12.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:11 smithi090 ceph-mon[43420]: osdmap e89: 12 total, 12 up, 12 in 2024-04-06T14:14:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 505.0 s, 1424.6 tps, lat 70.452 ms stddev 90.752 2024-04-06T14:14:13.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:12 smithi134 ceph-mon[47549]: pgmap v272: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:13.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:12 smithi105 ceph-mon[46969]: pgmap v272: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:12 smithi090 ceph-mon[43420]: pgmap v272: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:15.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:14 smithi134 ceph-mon[47549]: pgmap v273: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:15.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:14 smithi105 ceph-mon[46969]: pgmap v273: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:15.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:14 smithi090 ceph-mon[43420]: pgmap v273: 97 pgs: 97 active+clean; 8.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 31 MiB/s wr, 2.09k op/s 2024-04-06T14:14:17.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:16 smithi134 ceph-mon[47549]: pgmap v274: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 16 MiB/s wr, 716 op/s 2024-04-06T14:14:17.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:16 smithi105 ceph-mon[46969]: pgmap v274: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 16 MiB/s wr, 716 op/s 2024-04-06T14:14:17.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:16 smithi090 ceph-mon[43420]: pgmap v274: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 16 MiB/s wr, 716 op/s 2024-04-06T14:14:18.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 510.0 s, 2256.4 tps, lat 44.518 ms stddev 53.575 2024-04-06T14:14:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:18 smithi090 ceph-mon[43420]: osdmap e90: 12 total, 12 up, 12 in 2024-04-06T14:14:19.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:18 smithi090 ceph-mon[43420]: pgmap v276: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 20 MiB/s wr, 895 op/s 2024-04-06T14:14:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:18 smithi134 ceph-mon[47549]: osdmap e90: 12 total, 12 up, 12 in 2024-04-06T14:14:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:18 smithi134 ceph-mon[47549]: pgmap v276: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 20 MiB/s wr, 895 op/s 2024-04-06T14:14:19.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:18 smithi105 ceph-mon[46969]: osdmap e90: 12 total, 12 up, 12 in 2024-04-06T14:14:19.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:18 smithi105 ceph-mon[46969]: pgmap v276: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 20 MiB/s wr, 895 op/s 2024-04-06T14:14:21.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:20 smithi105 ceph-mon[46969]: pgmap v277: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 8.7 MiB/s wr, 367 op/s 2024-04-06T14:14:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:20 smithi090 ceph-mon[43420]: pgmap v277: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 8.7 MiB/s wr, 367 op/s 2024-04-06T14:14:21.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:20 smithi134 ceph-mon[47549]: pgmap v277: 97 pgs: 97 active+clean; 6.8 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 8.7 MiB/s wr, 367 op/s 2024-04-06T14:14:22.154 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:14:22.155 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:14:22.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:22 smithi090 ceph-mon[43420]: pgmap v278: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:22.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:22 smithi134 ceph-mon[47549]: pgmap v278: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:22.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:22 smithi105 ceph-mon[46969]: pgmap v278: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:22.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 -- 172.21.15.90:0/1959091046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc10f4b0 msgr2=0x7f4ddc107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 --2- 172.21.15.90:0/1959091046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc10f4b0 0x7f4ddc107db0 secure :-1 s=READY pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc01c0d0 tx=0x7f4dcc01c3e0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 -- 172.21.15.90:0/1959091046 shutdown_connections 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 --2- 172.21.15.90:0/1959091046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ddc11a590 0x7f4ddc11c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 --2- 172.21.15.90:0/1959091046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc10f4b0 0x7f4ddc107db0 secure :-1 s=CLOSED pgs=678 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc01c0d0 tx=0x7f4dcc01c3e0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 --2- 172.21.15.90:0/1959091046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc10f920 0x7f4ddc1082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.523+0000 7f4de23f3700 1 -- 172.21.15.90:0/1959091046 >> 172.21.15.90:0/1959091046 conn(0x7f4ddc06be00 msgr2=0x7f4ddc06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.525+0000 7f4de23f3700 1 -- 172.21.15.90:0/1959091046 shutdown_connections 2024-04-06T14:14:22.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.530+0000 7f4de23f3700 1 -- 172.21.15.90:0/1959091046 wait complete. 2024-04-06T14:14:22.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.530+0000 7f4de23f3700 1 Processor -- start 2024-04-06T14:14:22.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.530+0000 7f4de23f3700 1 -- start start 2024-04-06T14:14:22.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ddc10f920 0x7f4ddc119e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 0x7f4ddc115ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ddc1cc260 con 0x7f4ddc116400 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ddc1cc3d0 con 0x7f4ddc11a590 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de23f3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ddc1cc590 con 0x7f4ddc10f920 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de0990700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 0x7f4ddc115ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 0x7f4ddc115ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39908/0 (socket says 172.21.15.90:39908) 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/3244899380 learned_addr learned my addr 172.21.15.90:0/3244899380 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.531+0000 7f4de0990700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56352/0 (socket says 172.21.15.90:56352) 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.532+0000 7f4de0990700 1 -- 172.21.15.90:0/3244899380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ddc10f920 msgr2=0x7f4ddc119e00 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.532+0000 7f4de0990700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ddc10f920 0x7f4ddc119e00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.532+0000 7f4de0990700 1 -- 172.21.15.90:0/3244899380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 msgr2=0x7f4ddc115ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.532+0000 7f4de0990700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 0x7f4ddc115ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.532+0000 7f4de0990700 1 -- 172.21.15.90:0/3244899380 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4dcc01ad10 con 0x7f4ddc116400 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4de0990700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f4dd400b3c0 tx=0x7f4dd400b6d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4dd97fa700 1 -- 172.21.15.90:0/3244899380 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4011830 con 0x7f4ddc116400 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ddc1cc870 con 0x7f4ddc116400 2024-04-06T14:14:22.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ddc1ccd80 con 0x7f4ddc116400 2024-04-06T14:14:22.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4dd97fa700 1 -- 172.21.15.90:0/3244899380 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dd400be80 con 0x7f4ddc116400 2024-04-06T14:14:22.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.533+0000 7f4dd97fa700 1 -- 172.21.15.90:0/3244899380 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4010de0 con 0x7f4ddc116400 2024-04-06T14:14:22.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.537+0000 7f4dd97fa700 1 -- 172.21.15.90:0/3244899380 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4dd4010380 con 0x7f4ddc116400 2024-04-06T14:14:22.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.538+0000 7f4dd97fa700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 0x7f4dc80a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.538+0000 7f4dd97fa700 1 -- 172.21.15.90:0/3244899380 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4dd40cc590 con 0x7f4ddc116400 2024-04-06T14:14:22.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.540+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 0x7f4dc80a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.541+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 0x7f4dc80a3810 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc01c7b0 tx=0x7f4dcc01c040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 Processor -- start 2024-04-06T14:14:22.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 -- start start 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d6ab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d748a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573cc2990 con 0x55c573d6b0a0 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573cc0350 con 0x55c573d74de0 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573cbbc40 con 0x55c573d6a780 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4de0990700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4ddbfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d6ab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4ddbfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d6ab60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:56160/0 (socket says 172.21.15.90:56160) 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4de0990700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39916/0 (socket says 172.21.15.90:39916) 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.546+0000 7f4ddbfff700 1 -- 172.21.15.90:0/1603493813 learned_addr learned my addr 172.21.15.90:0/1603493813 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 -- 172.21.15.90:0/1603493813 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 msgr2=0x55c573d6ab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d6ab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 -- 172.21.15.90:0/1603493813 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 msgr2=0x55c573d748a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d748a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 -- 172.21.15.90:0/1603493813 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c573b3d3b0 con 0x55c573d74de0 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4de0990700 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40c9d70 tx=0x7f4dd4099340 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d2d6f6fdf28ac82e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4dd27fc700 1 -- 172.21.15.90:0/1603493813 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4082ea0 con 0x55c573d74de0 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4dd27fc700 1 -- 172.21.15.90:0/1603493813 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dd4099730 con 0x55c573d74de0 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.547+0000 7f4dd27fc700 1 -- 172.21.15.90:0/1603493813 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd40c9ee0 con 0x55c573d74de0 2024-04-06T14:14:22.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.549+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1603493813 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 msgr2=0x55c573d771d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.549+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40c9d70 tx=0x7f4dd4099340 comp rx=0 tx=0).stop 2024-04-06T14:14:22.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.551+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1603493813 shutdown_connections 2024-04-06T14:14:22.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.551+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d771d0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.551+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d6ab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.551+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1603493813 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d748a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.551+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1603493813 >> 172.21.15.90:0/1603493813 conn(0x55c573cc96a0 msgr2=0x55c573d65a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.553+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1603493813 shutdown_connections 2024-04-06T14:14:22.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.553+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1603493813 wait complete. 2024-04-06T14:14:22.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.553+0000 7f4dee5a7b80 1 -- start start 2024-04-06T14:14:22.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.553+0000 7f4dee5a7b80 1 Processor -- start 2024-04-06T14:14:22.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d748a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d723d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573c57d80 con 0x55c573d6b0a0 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573c57a20 con 0x55c573d74de0 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573b3f6a0 con 0x55c573d6a780 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4dee5a7b80 10 client.0 _ll_register_callbacks cb 0x55c573c61470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56370/0 (socket says 172.21.15.90:56370) 2024-04-06T14:14:22.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.554+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1499795526 learned_addr learned my addr 172.21.15.90:0/1499795526 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4de0990700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d723d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1499795526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 msgr2=0x55c573d748a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d748a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 msgr2=0x55c573d723d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d723d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1499795526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ddc1ccd80 con 0x55c573d6b0a0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.555+0000 7f4de0990700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d723d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f4dc40116c0 tx=0x7f4dc40119d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a5606d9225f41907 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dbb7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc4004520 con 0x55c573d6b0a0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4dc4003680 con 0x55c573d6b0a0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c573c7c380 con 0x55c573d6b0a0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4dc4003880 con 0x55c573d6b0a0 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dee5a7b80 10 client.16875 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:22.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c573ae9b60 con 0x55c573d6b0a0 2024-04-06T14:14:22.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.556+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dc4026020 con 0x55c573d6b0a0 2024-04-06T14:14:22.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4dc4021b90 con 0x55c573d6b0a0 2024-04-06T14:14:22.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 10 client.16875 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 10 client.16875 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 20 client.16875 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573d84b50 0x55c573d86f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 4 client.16875 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c573cb4700 con 0x55c573d84b50 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573d84b50 0x55c573d86f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573d84b50 0x55c573d86f70 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7f31faf1d7de2aab server_cookie=c05f77d9f5f0472f in_seq=0 out_seq=0 2024-04-06T14:14:22.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.557+0000 7f4dbb7fe700 10 client.16875 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.558+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1499795526 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c573cb4700 con 0x55c573d84b50 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.558+0000 7f4dbb7fe700 10 client.16875 handle_command_reply: tid=0 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 client.16875 shutdown 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 msgr2=0x55c573d71c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 secure :-1 s=READY pgs=680 cs=0 l=1 rev1=1 crypto rx=0x7f4dc40116c0 tx=0x7f4dc40119d0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 shutdown_connections 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d74de0 0x55c573d723d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573d84b50 0x55c573d86f70 crc :-1 s=READY pgs=161 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6a780 0x55c573d748a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.559+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1499795526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6b0a0 0x55c573d71c90 unknown :-1 s=CLOSED pgs=680 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.560+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 >> 172.21.15.90:0/1499795526 conn(0x55c573cc96a0 msgr2=0x55c573d64c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.560+0000 7f4dbb7fe700 0 client.16875 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:22.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.564+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 shutdown_connections 2024-04-06T14:14:22.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.564+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1499795526 wait complete. 2024-04-06T14:14:22.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.572+0000 7f4dee5a7b80 20 client.16875 trim_cache size 0 max 16384 2024-04-06T14:14:22.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.597+0000 7f4dee5a7b80 1 Processor -- start 2024-04-06T14:14:22.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.597+0000 7f4dee5a7b80 1 -- start start 2024-04-06T14:14:22.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.597+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6a780 0x55c573d771d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.597+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6b0a0 0x55c573d71570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.597+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d672b0 con 0x55c573d74de0 2024-04-06T14:14:22.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d74430 con 0x55c573d6a780 2024-04-06T14:14:22.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d701a0 con 0x55c573d6b0a0 2024-04-06T14:14:22.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4de0990700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4de0990700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:56380/0 (socket says 172.21.15.90:56380) 2024-04-06T14:14:22.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.598+0000 7f4de0990700 1 -- 172.21.15.90:0/297831702 learned_addr learned my addr 172.21.15.90:0/297831702 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 -- 172.21.15.90:0/297831702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6b0a0 msgr2=0x55c573d71570 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6b0a0 0x55c573d71570 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 -- 172.21.15.90:0/297831702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6a780 msgr2=0x55c573d771d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6a780 0x55c573d771d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 -- 172.21.15.90:0/297831702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c573c81ea0 con 0x55c573d74de0 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4de0990700 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40108f0 tx=0x7f4dd4004860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f069f49cb6d0937a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4dd2ffd700 1 -- 172.21.15.90:0/297831702 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd4082ea0 con 0x55c573d74de0 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4dd2ffd700 1 -- 172.21.15.90:0/297831702 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dd40a1bb0 con 0x55c573d74de0 2024-04-06T14:14:22.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.599+0000 7f4dd2ffd700 1 -- 172.21.15.90:0/297831702 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dd409d360 con 0x55c573d74de0 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.600+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/297831702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 msgr2=0x55c573d80aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.600+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 secure :-1 s=READY pgs=681 cs=0 l=1 rev1=1 crypto rx=0x7f4dd40108f0 tx=0x7f4dd4004860 comp rx=0 tx=0).stop 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.601+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/297831702 shutdown_connections 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.601+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d74de0 0x55c573d80aa0 unknown :-1 s=CLOSED pgs=681 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.601+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6a780 0x55c573d771d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.601+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/297831702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d6b0a0 0x55c573d71570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.601+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/297831702 >> 172.21.15.90:0/297831702 conn(0x55c573cc96a0 msgr2=0x55c573d61890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.604+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/297831702 shutdown_connections 2024-04-06T14:14:22.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.605+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/297831702 wait complete. 2024-04-06T14:14:22.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.605+0000 7f4dee5a7b80 1 -- start start 2024-04-06T14:14:22.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 Processor -- start 2024-04-06T14:14:22.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6a780 0x55c573d7b4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d74de0 0x55c573d79aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d7a080 con 0x55c573d6a780 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d7a1f0 con 0x55c573d6b0a0 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c573d7a3b0 con 0x55c573d74de0 2024-04-06T14:14:22.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4dee5a7b80 10 client.0 _ll_register_callbacks cb 0x55c573c61470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:22.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4ddb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39936/0 (socket says 172.21.15.90:39936) 2024-04-06T14:14:22.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1633444912 learned_addr learned my addr 172.21.15.90:0/1633444912 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.606+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6a780 0x55c573d7b4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1633444912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d74de0 msgr2=0x55c573d79aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d74de0 0x55c573d79aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1633444912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6a780 msgr2=0x55c573d7b4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6a780 0x55c573d7b4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 -- 172.21.15.90:0/1633444912 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c573b3d3b0 con 0x55c573d6b0a0 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.607+0000 7f4ddb7fe700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f4dc4001eb0 tx=0x7f4dc400faf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6e614c55b2318aae server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.610+0000 7f4dbb7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:22.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.610+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc40055a0 con 0x55c573d6b0a0 2024-04-06T14:14:22.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.610+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4dc4000e40 con 0x55c573d6b0a0 2024-04-06T14:14:22.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.612+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4dd4099730 con 0x55c573d6b0a0 2024-04-06T14:14:22.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.612+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4dc4005700 con 0x55c573d6b0a0 2024-04-06T14:14:22.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 10 client.26588 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:22.613 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c573b41cd0 con 0x55c573d6b0a0 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dd4099730 con 0x55c573d6b0a0 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4dc40215d0 con 0x55c573d6b0a0 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 10 client.26588 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 10 client.26588 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 20 client.26588 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573dd9b80 0x55c573ddbfa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 4 client.26588 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:14:22.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.613+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c573b0acf0 con 0x55c573dd9b80 2024-04-06T14:14:22.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.614+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573dd9b80 0x55c573ddbfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.615+0000 7f4dbb7fe700 10 client.26588 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:22.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4ddbfff700 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573dd9b80 0x55c573ddbfa0 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6a5502a63cef6b4c server_cookie=e6df3fbefa2ae92a in_seq=0 out_seq=0 2024-04-06T14:14:22.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4dbb7fe700 1 -- 172.21.15.90:0/1633444912 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55c573b0acf0 con 0x55c573dd9b80 2024-04-06T14:14:22.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4dbb7fe700 10 client.26588 handle_command_reply: tid=0 2024-04-06T14:14:22.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4dee5a7b80 1 client.26588 shutdown 2024-04-06T14:14:22.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 msgr2=0x55c573d7b9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.616+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f4dc4001eb0 tx=0x7f4dc400faf0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 shutdown_connections 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c573dd9b80 0x55c573ddbfa0 crc :-1 s=READY pgs=162 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c573d74de0 0x55c573d79aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c573d6a780 0x55c573d7b4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 --2- 172.21.15.90:0/1633444912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c573d6b0a0 0x55c573d7b9e0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 >> 172.21.15.90:0/1633444912 conn(0x55c573cc96a0 msgr2=0x55c573d74650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.617+0000 7f4dbb7fe700 0 client.26588 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:22.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.621+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 shutdown_connections 2024-04-06T14:14:22.621 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.621+0000 7f4dee5a7b80 1 -- 172.21.15.90:0/1633444912 wait complete. 2024-04-06T14:14:22.625 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.625+0000 7f4dee5a7b80 20 client.26588 trim_cache size 0 max 16384 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.625+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 msgr2=0x7f4dc80a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.625+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 0x7f4dc80a3810 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc01c7b0 tx=0x7f4dcc01c040 comp rx=0 tx=0).stop 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.625+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 msgr2=0x7f4ddc1168b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.626 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.625+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 secure :-1 s=READY pgs=679 cs=0 l=1 rev1=1 crypto rx=0x7f4dd400b3c0 tx=0x7f4dd400b6d0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 shutdown_connections 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4dc80a1350 0x7f4dc80a3810 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4ddc116400 0x7f4ddc1168b0 unknown :-1 s=CLOSED pgs=679 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4ddc10f920 0x7f4ddc119e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 --2- 172.21.15.90:0/3244899380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4ddc11a590 0x7f4ddc115ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.626+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 >> 172.21.15.90:0/3244899380 conn(0x7f4ddc06be00 msgr2=0x7f4ddc11a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.627 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.627+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 shutdown_connections 2024-04-06T14:14:22.628 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.627+0000 7f4de23f3700 1 -- 172.21.15.90:0/3244899380 wait complete. 2024-04-06T14:14:22.645 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:14:22.645 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:14:22.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/1651963352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 msgr2=0x7f8c340fff70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/1651963352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c340fff70 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f8c2400ba80 tx=0x7f8c2400bd90 comp rx=0 tx=0).stop 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/1651963352 shutdown_connections 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/1651963352 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c3410c9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/1651963352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 0x7f8c34105f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/1651963352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c340fff70 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.820+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/1651963352 >> 172.21.15.90:0/1651963352 conn(0x7f8c340fb600 msgr2=0x7f8c340fda20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:22.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.821+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/1651963352 shutdown_connections 2024-04-06T14:14:22.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.821+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/1651963352 wait complete. 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 Processor -- start 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 -- start start 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c34195440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 0x7f8c341997e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c34199e20 con 0x7f8c341004b0 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c34199f90 con 0x7f8c340ffaf0 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c3bd7b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3419a150 con 0x7f8c34105b90 2024-04-06T14:14:22.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.822+0000 7f8c39b17700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39b17700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:39964/0 (socket says 172.21.15.90:39964) 2024-04-06T14:14:22.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c34195440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39b17700 1 -- 172.21.15.90:0/614811401 learned_addr learned my addr 172.21.15.90:0/614811401 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:22.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c3a318700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 0x7f8c341997e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 -- 172.21.15.90:0/614811401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 msgr2=0x7f8c341997e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 0x7f8c341997e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 -- 172.21.15.90:0/614811401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 msgr2=0x7f8c34194f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 -- 172.21.15.90:0/614811401 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c2c009d00 con 0x7f8c341004b0 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39b17700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c39316700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c34195440 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f8c24018000 tx=0x7f8c24014b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.823+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c24010040 con 0x7f8c341004b0 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.824+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c2400b730 con 0x7f8c341004b0 2024-04-06T14:14:22.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.824+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c341a1a90 con 0x7f8c341004b0 2024-04-06T14:14:22.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.824+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8c2400ee80 con 0x7f8c341004b0 2024-04-06T14:14:22.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.824+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c2401fca0 con 0x7f8c341004b0 2024-04-06T14:14:22.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.827+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8c2401f440 con 0x7f8c341004b0 2024-04-06T14:14:22.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.827+0000 7f8c2affd700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 0x7f8c200a7ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:22.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.827+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8c240cdf10 con 0x7f8c341004b0 2024-04-06T14:14:22.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.830+0000 7f8c39b17700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 0x7f8c200a7ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:22.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.830+0000 7f8c39b17700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 0x7f8c200a7ca0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f8c30009ea0 tx=0x7f8c30009210 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:22.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.832+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c180052f0 con 0x7f8c341004b0 2024-04-06T14:14:22.838 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:22.837+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8c24097370 con 0x7f8c341004b0 2024-04-06T14:14:23.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 515.0 s, 1066.4 tps, lat 93.475 ms stddev 155.036 2024-04-06T14:14:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.217+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8c18005160 con 0x7f8c341004b0 2024-04-06T14:14:23.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.218+0000 7f8c2affd700 1 -- 172.21.15.90:0/614811401 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f8c24023090 con 0x7f8c341004b0 2024-04-06T14:14:23.219 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:14:23.219 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:14:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 msgr2=0x7f8c200a7ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 0x7f8c200a7ca0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f8c30009ea0 tx=0x7f8c30009210 comp rx=0 tx=0).stop 2024-04-06T14:14:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 msgr2=0x7f8c34195440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c34195440 secure :-1 s=READY pgs=682 cs=0 l=1 rev1=1 crypto rx=0x7f8c24018000 tx=0x7f8c24014b40 comp rx=0 tx=0).stop 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 shutdown_connections 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8c200a57e0 0x7f8c200a7ca0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8c34105b90 0x7f8c341997e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c340ffaf0 0x7f8c34194f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 --2- 172.21.15.90:0/614811401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8c341004b0 0x7f8c34195440 unknown :-1 s=CLOSED pgs=682 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 >> 172.21.15.90:0/614811401 conn(0x7f8c340fb600 msgr2=0x7f8c340fbe00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 shutdown_connections 2024-04-06T14:14:23.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.221+0000 7f8c3bd7b700 1 -- 172.21.15.90:0/614811401 wait complete. 2024-04-06T14:14:23.223 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:14:23.236 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:23.236 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:14:23.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.432+0000 7fd46e396700 1 --2- 172.21.15.90:0/2800977130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468104f80 0x7fd46810cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 -- 172.21.15.90:0/2800977130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468100820 msgr2=0x7fd468104850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/2800977130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468100820 0x7fd468104850 secure :-1 s=READY pgs=683 cs=0 l=1 rev1=1 crypto rx=0x7fd458009990 tx=0x7fd458009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 -- 172.21.15.90:0/2800977130 shutdown_connections 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/2800977130 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468104f80 0x7fd46810cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/2800977130 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd468100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/2800977130 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468100820 0x7fd468104850 unknown :-1 s=CLOSED pgs=683 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 -- 172.21.15.90:0/2800977130 >> 172.21.15.90:0/2800977130 conn(0x7fd468076d40 msgr2=0x7fd468077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 -- 172.21.15.90:0/2800977130 shutdown_connections 2024-04-06T14:14:23.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.433+0000 7fd46fdf9700 1 -- 172.21.15.90:0/2800977130 wait complete. 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 Processor -- start 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 -- start start 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd46819e040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 0x7fd468199af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd46819a150 con 0x7fd468104f80 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd46819a2c0 con 0x7fd468199670 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.434+0000 7fd46fdf9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd46819a480 con 0x7fd4680ffe70 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46d394700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46d394700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39380/0 (socket says 172.21.15.90:39380) 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46d394700 1 -- 172.21.15.90:0/565558775 learned_addr learned my addr 172.21.15.90:0/565558775 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46e396700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 0x7fd468199af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.437 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd46819e040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 -- 172.21.15.90:0/565558775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 msgr2=0x7fd468199af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 0x7fd468199af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 -- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 msgr2=0x7fd46819e580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 -- 172.21.15.90:0/565558775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd460009c40 con 0x7fd4680ffe70 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46d394700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46e396700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 0x7fd468199af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46db95700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd46819e040 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fd46400d950 tx=0x7fd46400dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd45effd700 1 -- 172.21.15.90:0/565558775 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd464009930 con 0x7fd4680ffe70 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4580095f0 con 0x7fd4680ffe70 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd45effd700 1 -- 172.21.15.90:0/565558775 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd46400be70 con 0x7fd4680ffe70 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd45effd700 1 -- 172.21.15.90:0/565558775 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd46400f3c0 con 0x7fd4680ffe70 2024-04-06T14:14:23.438 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.435+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4681a6420 con 0x7fd4680ffe70 2024-04-06T14:14:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 Processor -- start 2024-04-06T14:14:23.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd45effd700 1 -- 172.21.15.90:0/565558775 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fd464010560 con 0x7fd4680ffe70 2024-04-06T14:14:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 -- start start 2024-04-06T14:14:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 0x5651af28fb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a49e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af1e7990 con 0x5651af2a25f0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af1e5350 con 0x5651af2900a0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.440+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af1e0c40 con 0x5651af28f780 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46db95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a49e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46db95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53998/0 (socket says 172.21.15.90:53998) 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46db95700 1 -- 172.21.15.90:0/523101352 learned_addr learned my addr 172.21.15.90:0/523101352 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46e396700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 0x5651af28fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 -- 172.21.15.90:0/523101352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 msgr2=0x5651af28fb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 0x5651af28fb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 -- 172.21.15.90:0/523101352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 msgr2=0x5651af2a20b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 -- 172.21.15.90:0/523101352 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5651af0623b0 con 0x5651af2a25f0 2024-04-06T14:14:23.444 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46e396700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 0x5651af28fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46db95700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd45effd700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 0x7fd4540a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46d394700 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a49e0 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7fd45800b2e0 tx=0x7fd458005e50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=15e5966ad056940b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd46e396700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 0x7fd4540a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd45effd700 1 -- 172.21.15.90:0/565558775 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fd4640cc840 con 0x7fd4680ffe70 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd453fff700 1 -- 172.21.15.90:0/523101352 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd45801d050 con 0x5651af2a25f0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd453fff700 1 -- 172.21.15.90:0/523101352 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd45801e040 con 0x5651af2a25f0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.441+0000 7fd453fff700 1 -- 172.21.15.90:0/523101352 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd458003d90 con 0x5651af2a25f0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 -- 172.21.15.90:0/523101352 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 msgr2=0x5651af2a49e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd46e396700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 0x7fd4540a37d0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fd460009fd0 tx=0x7fd46000b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a49e0 secure :-1 s=READY pgs=684 cs=0 l=1 rev1=1 crypto rx=0x7fd45800b2e0 tx=0x7fd458005e50 comp rx=0 tx=0).stop 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 -- 172.21.15.90:0/523101352 shutdown_connections 2024-04-06T14:14:23.445 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a49e0 unknown :-1 s=CLOSED pgs=684 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af28f780 0x5651af28fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/523101352 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a20b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 -- 172.21.15.90:0/523101352 >> 172.21.15.90:0/523101352 conn(0x5651af1ee6a0 msgr2=0x5651af28aa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 -- 172.21.15.90:0/523101352 shutdown_connections 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.442+0000 7fd47bfadb80 1 -- 172.21.15.90:0/523101352 wait complete. 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 -- start start 2024-04-06T14:14:23.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 Processor -- start 2024-04-06T14:14:23.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29f320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af17cd80 con 0x5651af2a25f0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af17ca20 con 0x5651af28f780 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af0646a0 con 0x5651af2900a0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd47bfadb80 10 client.0 _ll_register_callbacks cb 0x5651af186470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46e396700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46db95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46e396700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39390/0 (socket says 172.21.15.90:39390) 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46db95700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54012/0 (socket says 172.21.15.90:54012) 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46e396700 1 -- 172.21.15.90:0/3904489464 learned_addr learned my addr 172.21.15.90:0/3904489464 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.443+0000 7fd46d394700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29f320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 -- 172.21.15.90:0/3904489464 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 msgr2=0x5651af29f320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29f320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 -- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 msgr2=0x5651af2a2030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 -- 172.21.15.90:0/3904489464 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4681a6420 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46db95700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd46e396700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7fd460009230 tx=0x7fd460004b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e8f29f3e7da2c4a6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd4517fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd46002b660 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fd460016e30 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5651af1a1380 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd460009090 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd47bfadb80 10 client.16899 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.444+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fd46810d370 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.445+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5651af1a1380 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.445+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fd46001e070 con 0x5651af2a25f0 2024-04-06T14:14:23.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.446+0000 7fd47bfadb80 10 client.16899 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.446+0000 7fd47bfadb80 10 client.16899 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.446+0000 7fd47bfadb80 20 client.16899 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af2b2190 0x5651af2b45b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd47bfadb80 4 client.16899 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5651af1d9700 con 0x5651af2b2190 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd46db95700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af2b2190 0x5651af2b45b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd46db95700 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af2b2190 0x5651af2b45b0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=26425fff064fe741 server_cookie=46070b1ed8bb20be in_seq=0 out_seq=0 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.447+0000 7fd4517fa700 10 client.16899 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd4517fa700 1 -- 172.21.15.90:0/3904489464 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5651af1d9700 con 0x5651af2b2190 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd4517fa700 10 client.16899 handle_command_reply: tid=0 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 client.16899 shutdown 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 msgr2=0x5651af29fa60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 secure :-1 s=READY pgs=685 cs=0 l=1 rev1=1 crypto rx=0x7fd460009230 tx=0x7fd460004b40 comp rx=0 tx=0).stop 2024-04-06T14:14:23.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 shutdown_connections 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af2b2190 0x5651af2b45b0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29fa60 unknown :-1 s=CLOSED pgs=685 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd4517fa700 0 client.16899 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a2030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3904489464 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29f320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 >> 172.21.15.90:0/3904489464 conn(0x5651af1ee6a0 msgr2=0x5651af289c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 shutdown_connections 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3904489464 wait complete. 2024-04-06T14:14:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.448+0000 7fd47bfadb80 20 client.16899 trim_cache size 0 max 16384 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 Processor -- start 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 -- start start 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a9210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29dbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af2a9750 con 0x5651af2a25f0 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af29d870 con 0x5651af28f780 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.468+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af2a08f0 con 0x5651af2900a0 2024-04-06T14:14:23.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39402/0 (socket says 172.21.15.90:39402) 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 -- 172.21.15.90:0/3071784891 learned_addr learned my addr 172.21.15.90:0/3071784891 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 -- 172.21.15.90:0/3071784891 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 msgr2=0x5651af29dbf0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29dbf0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 -- 172.21.15.90:0/3071784891 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 msgr2=0x5651af2a9210 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a9210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 -- 172.21.15.90:0/3071784891 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5651af00eb60 con 0x5651af2a25f0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.470+0000 7fd46e396700 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7fd46000f010 tx=0x7fd460007780 comp rx=0 tx=0).ready entity=mon.0 client_cookie=37c5e17e6197a693 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.471+0000 7fd45cff9700 1 -- 172.21.15.90:0/3071784891 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd460016e30 con 0x5651af2a25f0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.471+0000 7fd45cff9700 1 -- 172.21.15.90:0/3071784891 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd46000f040 con 0x5651af2a25f0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.471+0000 7fd45cff9700 1 -- 172.21.15.90:0/3071784891 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd46002bd90 con 0x5651af2a25f0 2024-04-06T14:14:23.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.472+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3071784891 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 msgr2=0x5651af29e130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.472+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 secure :-1 s=READY pgs=686 cs=0 l=1 rev1=1 crypto rx=0x7fd46000f010 tx=0x7fd460007780 comp rx=0 tx=0).stop 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.476+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3071784891 shutdown_connections 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.476+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af29e130 unknown :-1 s=CLOSED pgs=686 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.476+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af28f780 0x5651af2a9210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.476+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/3071784891 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2900a0 0x5651af29dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.476+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3071784891 >> 172.21.15.90:0/3071784891 conn(0x5651af1ee6a0 msgr2=0x5651af29a610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.482+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3071784891 shutdown_connections 2024-04-06T14:14:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.482+0000 7fd47bfadb80 1 -- 172.21.15.90:0/3071784891 wait complete. 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.482+0000 7fd47bfadb80 1 -- start start 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 Processor -- start 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a9000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2a7b60 0x5651af2a7fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af305fd0 con 0x5651af2a25f0 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af306110 con 0x5651af2900a0 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5651af3062d0 con 0x5651af2a7b60 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd47bfadb80 10 client.0 _ll_register_callbacks cb 0x5651af186470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.483+0000 7fd46d394700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39408/0 (socket says 172.21.15.90:39408) 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 -- 172.21.15.90:0/2551650536 learned_addr learned my addr 172.21.15.90:0/2551650536 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 -- 172.21.15.90:0/2551650536 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2a7b60 msgr2=0x5651af2a7fe0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2a7b60 0x5651af2a7fe0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 -- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 msgr2=0x5651af2a9000 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a9000 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.484+0000 7fd46d394700 1 -- 172.21.15.90:0/2551650536 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd46810d370 con 0x5651af2a25f0 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.486+0000 7fd46d394700 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7fd458026040 tx=0x7fd458003e10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=159e75a356010397 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.486+0000 7fd4517fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.487+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd45800b880 con 0x5651af2a25f0 2024-04-06T14:14:23.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.487+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fd4580041c0 con 0x5651af2a25f0 2024-04-06T14:14:23.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.488+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fd45801e040 con 0x5651af2a25f0 2024-04-06T14:14:23.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.490+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd45801d020 con 0x5651af2a25f0 2024-04-06T14:14:23.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.492+0000 7fd47bfadb80 10 client.16905 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:23.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.492+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5651af1a6ea0 con 0x5651af2a25f0 2024-04-06T14:14:23.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd45801e040 con 0x5651af2a25f0 2024-04-06T14:14:23.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fd458023b50 con 0x5651af2a25f0 2024-04-06T14:14:23.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 10 client.16905 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 10 client.16905 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 20 client.16905 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af3075a0 0x5651af309990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 4 client.16905 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5651af02fcf0 con 0x5651af3075a0 2024-04-06T14:14:23.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.493+0000 7fd46db95700 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af3075a0 0x5651af309990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd4517fa700 10 client.16905 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:23.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd46db95700 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af3075a0 0x5651af309990 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=43ac0e8d1b2fa7c5 server_cookie=b0978982172e07e8 in_seq=0 out_seq=0 2024-04-06T14:14:23.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd4517fa700 1 -- 172.21.15.90:0/2551650536 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5651af02fcf0 con 0x5651af3075a0 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd4517fa700 10 client.16905 handle_command_reply: tid=0 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd47bfadb80 1 client.16905 shutdown 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 msgr2=0x5651af2a7620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.494+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 secure :-1 s=READY pgs=687 cs=0 l=1 rev1=1 crypto rx=0x7fd458026040 tx=0x7fd458003e10 comp rx=0 tx=0).stop 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 shutdown_connections 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5651af3075a0 0x5651af309990 crc :-1 s=READY pgs=62 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5651af2a7b60 0x5651af2a7fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5651af2900a0 0x5651af2a9000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 --2- 172.21.15.90:0/2551650536 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5651af2a25f0 0x5651af2a7620 unknown :-1 s=CLOSED pgs=687 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 >> 172.21.15.90:0/2551650536 conn(0x5651af1ee6a0 msgr2=0x5651af28a4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd4517fa700 0 client.16905 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 shutdown_connections 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 1 -- 172.21.15.90:0/2551650536 wait complete. 2024-04-06T14:14:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd47bfadb80 20 client.16905 trim_cache size 0 max 16384 2024-04-06T14:14:23.497 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 msgr2=0x7fd4540a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 0x7fd4540a37d0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fd460009fd0 tx=0x7fd46000b040 comp rx=0 tx=0).stop 2024-04-06T14:14:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 msgr2=0x7fd46819e040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.495+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd46819e040 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fd46400d950 tx=0x7fd46400dc60 comp rx=0 tx=0).stop 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 shutdown_connections 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd4540a1310 0x7fd4540a37d0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd468199670 0x7fd468199af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd4680ffe70 0x7fd46819e040 secure :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7fd46400d950 tx=0x7fd46400dc60 comp rx=0 tx=0).stop 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 --2- 172.21.15.90:0/565558775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd468104f80 0x7fd46819e580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.496+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 >> 172.21.15.90:0/565558775 conn(0x7fd468076d40 msgr2=0x7fd4680fe770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.500+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 shutdown_connections 2024-04-06T14:14:23.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.500+0000 7fd46fdf9700 1 -- 172.21.15.90:0/565558775 wait complete. 2024-04-06T14:14:23.515 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:23.516 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:14:23.717 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3522325271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b411a590 msgr2=0x7ff1b411c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3522325271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b411a590 0x7ff1b411c980 secure :-1 s=READY pgs=688 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac01a9a0 tx=0x7ff1ac01acb0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3522325271 shutdown_connections 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3522325271 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b411a590 0x7ff1b411c980 unknown :-1 s=CLOSED pgs=688 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3522325271 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3522325271 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1b410f920 0x7ff1b41082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.718 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.716+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3522325271 >> 172.21.15.90:0/3522325271 conn(0x7ff1b406be00 msgr2=0x7ff1b406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.726+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3522325271 shutdown_connections 2024-04-06T14:14:23.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.727+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3522325271 wait complete. 2024-04-06T14:14:23.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.727+0000 7ff1ba59b700 1 Processor -- start 2024-04-06T14:14:23.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 -- start start 2024-04-06T14:14:23.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1b410f920 0x7ff1b4119fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b411a590 0x7ff1b4115d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1b4116550 con 0x7ff1b410f4b0 2024-04-06T14:14:23.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1b41166c0 con 0x7ff1b411a590 2024-04-06T14:14:23.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.733+0000 7ff1ba59b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1b4116880 con 0x7ff1b410f920 2024-04-06T14:14:23.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.735+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.735+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39430/0 (socket says 172.21.15.90:39430) 2024-04-06T14:14:23.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.735+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3776247099 learned_addr learned my addr 172.21.15.90:0/3776247099 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.735+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b411a590 0x7ff1b4115d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3776247099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1b410f920 msgr2=0x7ff1b4119fd0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:14:23.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1b410f920 0x7ff1b4119fd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3776247099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b411a590 msgr2=0x7ff1b4115d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b411a590 0x7ff1b4115d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3776247099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1ac01a600 con 0x7ff1b410f4b0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.736+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7ff19c00b550 tx=0x7ff19c00b860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.737+0000 7ff1b17fa700 1 -- 172.21.15.90:0/3776247099 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c011830 con 0x7ff1b410f4b0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.737+0000 7ff1b17fa700 1 -- 172.21.15.90:0/3776247099 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff19c011e50 con 0x7ff1b410f4b0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.737+0000 7ff1b17fa700 1 -- 172.21.15.90:0/3776247099 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c00f2e0 con 0x7ff1b410f4b0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.737+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff1b41aa220 con 0x7ff1b410f4b0 2024-04-06T14:14:23.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.737+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1b41aa650 con 0x7ff1b410f4b0 2024-04-06T14:14:23.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.742+0000 7ff1c674fb80 1 Processor -- start 2024-04-06T14:14:23.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 -- start start 2024-04-06T14:14:23.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8f0b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8fa8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c848990 con 0x56193c8fade0 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c846350 con 0x56193c8f0780 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.747+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c841c40 con 0x56193c8f10a0 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b17fa700 1 -- 172.21.15.90:0/3776247099 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff19c011990 con 0x7ff1b410f4b0 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39432/0 (socket says 172.21.15.90:39432) 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 -- 172.21.15.90:0/4163985190 learned_addr learned my addr 172.21.15.90:0/4163985190 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.749 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8f0b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 -- 172.21.15.90:0/4163985190 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 msgr2=0x56193c8fa8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8fa8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8fa8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 -- 172.21.15.90:0/4163985190 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 msgr2=0x56193c8f0b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8f0b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 -- 172.21.15.90:0/4163985190 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56193c6c33b0 con 0x56193c8fade0 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8f0b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:23.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.748+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7ff1a4009da0 tx=0x7ff1a400b730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=169fdef9d950bb20 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.751+0000 7ff1b17fa700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 0x7ff1a00a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.751+0000 7ff1b17fa700 1 -- 172.21.15.90:0/3776247099 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff19c0cc590 con 0x7ff1b410f4b0 2024-04-06T14:14:23.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.751+0000 7ff1aa7fc700 1 -- 172.21.15.90:0/4163985190 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1a4013410 con 0x56193c8fade0 2024-04-06T14:14:23.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.751+0000 7ff1aa7fc700 1 -- 172.21.15.90:0/4163985190 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1a4013a30 con 0x56193c8fade0 2024-04-06T14:14:23.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.751+0000 7ff1aa7fc700 1 -- 172.21.15.90:0/4163985190 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1a4012a10 con 0x56193c8fade0 2024-04-06T14:14:23.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.752+0000 7ff1c674fb80 1 -- 172.21.15.90:0/4163985190 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 msgr2=0x56193c8fd1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.752+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 secure :-1 s=READY pgs=690 cs=0 l=1 rev1=1 crypto rx=0x7ff1a4009da0 tx=0x7ff1a400b730 comp rx=0 tx=0).stop 2024-04-06T14:14:23.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.754+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 0x7ff1a00a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.754+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 0x7ff1a00a3960 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac01c5d0 tx=0x7ff1ac02a040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 -- 172.21.15.90:0/4163985190 shutdown_connections 2024-04-06T14:14:23.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c8fd1d0 unknown :-1 s=CLOSED pgs=690 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8f0b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/4163985190 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8fa8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 -- 172.21.15.90:0/4163985190 >> 172.21.15.90:0/4163985190 conn(0x56193c84f6a0 msgr2=0x56193c8eba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 -- 172.21.15.90:0/4163985190 shutdown_connections 2024-04-06T14:14:23.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.755+0000 7ff1c674fb80 1 -- 172.21.15.90:0/4163985190 wait complete. 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 -- start start 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 Processor -- start 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8f7be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8fade0 0x56193c8f81f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c7ddd80 con 0x56193c8f0780 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c7dda20 con 0x56193c8fade0 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c6c56a0 con 0x56193c8f10a0 2024-04-06T14:14:23.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.756+0000 7ff1c674fb80 10 client.0 _ll_register_callbacks cb 0x56193c7e7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:23.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.757+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.757+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39446/0 (socket says 172.21.15.90:39446) 2024-04-06T14:14:23.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.757+0000 7ff1b3fff700 1 -- 172.21.15.90:0/1800623570 learned_addr learned my addr 172.21.15.90:0/1800623570 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.758+0000 7ff1b8b38700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8fade0 0x56193c8f81f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 -- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 msgr2=0x56193c8f7be0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8f7be0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 -- 172.21.15.90:0/1800623570 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8fade0 msgr2=0x56193c8f81f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8fade0 0x56193c8f81f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 -- 172.21.15.90:0/1800623570 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1a40099d0 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7ff19c0961a0 tx=0x7ff19c0993a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=eb8263c34cc16313 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.759+0000 7ff1937fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c09d560 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff19c0a1bb0 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56193c802380 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff19c095e40 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 10 client.16917 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56193c66fb60 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56193c802380 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff19c099990 con 0x56193c8f0780 2024-04-06T14:14:23.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 10 client.16917 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 10 client.16917 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.760+0000 7ff1c674fb80 20 client.16917 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.761+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c90a9f0 0x56193c90ce10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.761+0000 7ff1c674fb80 4 client.16917 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.761+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56193c83a700 con 0x56193c90a9f0 2024-04-06T14:14:23.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.762+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c90a9f0 0x56193c90ce10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.762+0000 7ff1937fe700 10 client.16917 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:23.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.762+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c90a9f0 0x56193c90ce10 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=7d7e2eacd85dcdfa server_cookie=4cf31e86a6e6ba5 in_seq=0 out_seq=0 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.763+0000 7ff1937fe700 1 -- 172.21.15.90:0/1800623570 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56193c83a700 con 0x56193c90a9f0 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.763+0000 7ff1937fe700 10 client.16917 handle_command_reply: tid=0 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.763+0000 7ff1c674fb80 1 client.16917 shutdown 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 msgr2=0x56193c8fa7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 secure :-1 s=READY pgs=691 cs=0 l=1 rev1=1 crypto rx=0x7ff19c0961a0 tx=0x7ff19c0993a0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 shutdown_connections 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c90a9f0 0x56193c90ce10 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8fade0 0x56193c8f81f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f0780 0x56193c8fa7b0 unknown :-1 s=CLOSED pgs=691 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1937fe700 0 client.16917 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/1800623570 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8f7be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 >> 172.21.15.90:0/1800623570 conn(0x56193c84f6a0 msgr2=0x56193c8eac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 shutdown_connections 2024-04-06T14:14:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.764+0000 7ff1c674fb80 1 -- 172.21.15.90:0/1800623570 wait complete. 2024-04-06T14:14:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.766+0000 7ff1c674fb80 20 client.16917 trim_cache size 0 max 16384 2024-04-06T14:14:23.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:23 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/614811401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:23.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.787+0000 7ff1c674fb80 1 Processor -- start 2024-04-06T14:14:23.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 -- start start 2024-04-06T14:14:23.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f10a0 0x56193c8f2630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8fade0 0x56193c8fa2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c8f1a40 con 0x56193c8f10a0 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c8f0ef0 con 0x56193c8f0780 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c8ec680 con 0x56193c8fade0 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1b37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f10a0 0x56193c8f2630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54052/0 (socket says 172.21.15.90:54052) 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.788+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3066050739 learned_addr learned my addr 172.21.15.90:0/3066050739 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3066050739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8fade0 msgr2=0x56193c8fa2e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8fade0 0x56193c8fa2e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3066050739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f10a0 msgr2=0x56193c8f2630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f10a0 0x56193c8f2630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3066050739 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56193c807ea0 con 0x56193c8f0780 2024-04-06T14:14:23.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7ff19c099430 tx=0x7ff19c097ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bf1ad419784d2d16 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1aaffd700 1 -- 172.21.15.90:0/3066050739 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c09a550 con 0x56193c8f0780 2024-04-06T14:14:23.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1aaffd700 1 -- 172.21.15.90:0/3066050739 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff19c0a1bb0 con 0x56193c8f0780 2024-04-06T14:14:23.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.789+0000 7ff1aaffd700 1 -- 172.21.15.90:0/3066050739 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c095e40 con 0x56193c8f0780 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3066050739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 msgr2=0x56193c8ebc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7ff19c099430 tx=0x7ff19c097ce0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3066050739 shutdown_connections 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8fade0 0x56193c8fa2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c8ebc30 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3066050739 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8f10a0 0x56193c8f2630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3066050739 >> 172.21.15.90:0/3066050739 conn(0x56193c84f6a0 msgr2=0x56193c8eb520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3066050739 shutdown_connections 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3066050739 wait complete. 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.790+0000 7ff1c674fb80 1 -- start start 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 Processor -- start 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8ffbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c900120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c902090 con 0x56193c8fade0 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c900660 con 0x56193c8f0780 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56193c900820 con 0x56193c8f10a0 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1c674fb80 10 client.0 _ll_register_callbacks cb 0x56193c7e7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:23.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1b3fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54054/0 (socket says 172.21.15.90:54054) 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1b8b38700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c900120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3191146618 learned_addr learned my addr 172.21.15.90:0/3191146618 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.791+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8ffbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 msgr2=0x56193c8ffbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8ffbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3191146618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 msgr2=0x56193c900120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c900120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 -- 172.21.15.90:0/3191146618 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56193c66fb60 con 0x56193c8f0780 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8ffbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1b3fff700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7ff19c097a80 tx=0x7ff19c099c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7a2ca938ab82db0b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:23.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19c010f10 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff19c098d20 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff1a4013a30 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff19c09a4d0 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 10 client.26618 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56193c6c7cd0 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff1a4013a30 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff19c0a7020 con 0x56193c8f0780 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 10 client.26618 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 10 client.26618 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.792+0000 7ff1c674fb80 20 client.26618 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c95f6d0 0x56193c961ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:23.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1c674fb80 4 client.26618 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:23.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x56193c690cf0 con 0x56193c95f6d0 2024-04-06T14:14:23.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c95f6d0 0x56193c961ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:23.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1937fe700 10 client.26618 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:23.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1b37fe700 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c95f6d0 0x56193c961ac0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c0055284bac6fd41 server_cookie=69457ef57e1007b1 in_seq=0 out_seq=0 2024-04-06T14:14:23.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1937fe700 1 -- 172.21.15.90:0/3191146618 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x56193c690cf0 con 0x56193c95f6d0 2024-04-06T14:14:23.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1937fe700 10 client.26618 handle_command_reply: tid=0 2024-04-06T14:14:23.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.793+0000 7ff1c674fb80 1 client.26618 shutdown 2024-04-06T14:14:23.798 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:23.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 msgr2=0x56193c9017b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7ff19c097a80 tx=0x7ff19c099c20 comp rx=0 tx=0).stop 2024-04-06T14:14:23.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 shutdown_connections 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x56193c95f6d0 0x56193c961ac0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56193c8fade0 0x56193c900120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56193c8f0780 0x56193c9017b0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 --2- 172.21.15.90:0/3191146618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56193c8f10a0 0x56193c8ffbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1937fe700 0 client.26618 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 >> 172.21.15.90:0/3191146618 conn(0x56193c84f6a0 msgr2=0x56193c8ef770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 shutdown_connections 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 1 -- 172.21.15.90:0/3191146618 wait complete. 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1c674fb80 20 client.26618 trim_cache size 0 max 16384 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 msgr2=0x7ff1a00a3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 0x7ff1a00a3960 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7ff1ac01c5d0 tx=0x7ff1ac02a040 comp rx=0 tx=0).stop 2024-04-06T14:14:23.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 msgr2=0x7ff1b4119a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 secure :-1 s=READY pgs=689 cs=0 l=1 rev1=1 crypto rx=0x7ff19c00b550 tx=0x7ff19c00b860 comp rx=0 tx=0).stop 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 shutdown_connections 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff1a00a14a0 0x7ff1a00a3960 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff1b411a590 0x7ff1b4115d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff1b410f4b0 0x7ff1b4119a90 unknown :-1 s=CLOSED pgs=689 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 --2- 172.21.15.90:0/3776247099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff1b410f920 0x7ff1b4119fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.794+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 >> 172.21.15.90:0/3776247099 conn(0x7ff1b406be00 msgr2=0x7ff1b411bdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.795+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 shutdown_connections 2024-04-06T14:14:23.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:23.795+0000 7ff1ba59b700 1 -- 172.21.15.90:0/3776247099 wait complete. 2024-04-06T14:14:23.811 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:23.811 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:14:24.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 -- 172.21.15.90:0/2613449913 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073150 msgr2=0x7f83a0073530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 --2- 172.21.15.90:0/2613449913 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073150 0x7f83a0073530 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f8394009990 tx=0x7f8394009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 -- 172.21.15.90:0/2613449913 shutdown_connections 2024-04-06T14:14:24.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 --2- 172.21.15.90:0/2613449913 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 0x7f83a0114100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 --2- 172.21.15.90:0/2613449913 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073150 0x7f83a0073530 secure :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f8394009990 tx=0x7f8394009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 --2- 172.21.15.90:0/2613449913 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a0073a70 0x7f83a01117d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.038+0000 7f83a71d9700 1 -- 172.21.15.90:0/2613449913 >> 172.21.15.90:0/2613449913 conn(0x7f83a00fc7f0 msgr2=0x7f83a00fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.040+0000 7f83a71d9700 1 -- 172.21.15.90:0/2613449913 shutdown_connections 2024-04-06T14:14:24.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.040+0000 7f83a71d9700 1 -- 172.21.15.90:0/2613449913 wait complete. 2024-04-06T14:14:24.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.041+0000 7f83a71d9700 1 Processor -- start 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.041+0000 7f83a71d9700 1 -- start start 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073a70 0x7f83a01a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 0x7f83a01a3d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83a019e3d0 con 0x7f83a0111d10 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83a019e540 con 0x7f83a0073a70 2024-04-06T14:14:24.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a71d9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83a019e700 con 0x7f83a019d950 2024-04-06T14:14:24.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51398/0 (socket says 172.21.15.90:51398) 2024-04-06T14:14:24.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 -- 172.21.15.90:0/859322660 learned_addr learned my addr 172.21.15.90:0/859322660 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f839ffff700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 0x7f83a01a3d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a4f75700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073a70 0x7f83a01a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 -- 172.21.15.90:0/859322660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073a70 msgr2=0x7f83a01a37d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073a70 0x7f83a01a37d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 -- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 msgr2=0x7f83a01a3d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 0x7f83a01a3d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.042+0000 7f83a5776700 1 -- 172.21.15.90:0/859322660 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83940095f0 con 0x7f83a019d950 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f83a5776700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f839800b9e0 tx=0x7f839800bda0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f839dffb700 1 -- 172.21.15.90:0/859322660 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f839800c9b0 con 0x7f83a019d950 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f839dffb700 1 -- 172.21.15.90:0/859322660 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8398014440 con 0x7f83a019d950 2024-04-06T14:14:24.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f839dffb700 1 -- 172.21.15.90:0/859322660 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f83980125a0 con 0x7f83a019d950 2024-04-06T14:14:24.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83a01aa6a0 con 0x7f83a019d950 2024-04-06T14:14:24.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.043+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83a01aab30 con 0x7f83a019d950 2024-04-06T14:14:24.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.045+0000 7f839dffb700 1 -- 172.21.15.90:0/859322660 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f83980145b0 con 0x7f83a019d950 2024-04-06T14:14:24.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.046+0000 7f839dffb700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 0x7f83900a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.046+0000 7f839dffb700 1 -- 172.21.15.90:0/859322660 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f83980cc2b0 con 0x7f83a019d950 2024-04-06T14:14:24.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.046+0000 7f83a4f75700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 0x7f83900a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.047+0000 7f83a4f75700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 0x7f83900a3850 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8394000c00 tx=0x7f839400b580 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.047+0000 7f83b338db80 1 Processor -- start 2024-04-06T14:14:24.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 -- start start 2024-04-06T14:14:24.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7ad8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7b01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f6fb990 con 0x55a23f7a3780 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f6f9350 con 0x55a23f7a40a0 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83b338db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f6f4c40 con 0x55a23f7adde0 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83a4f75700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83a4f75700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39480/0 (socket says 172.21.15.90:39480) 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83a4f75700 1 -- 172.21.15.90:0/3377109283 learned_addr learned my addr 172.21.15.90:0/3377109283 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7ad8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83a5776700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7b01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 -- 172.21.15.90:0/3377109283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 msgr2=0x55a23f7b01d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7b01d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 -- 172.21.15.90:0/3377109283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 msgr2=0x55a23f7a3b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f839ffff700 1 -- 172.21.15.90:0/3377109283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a23f5763b0 con 0x55a23f7a40a0 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.048+0000 7f83a4f75700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.049+0000 7f839ffff700 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7ad8a0 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f838c00bc40 tx=0x7f838c00bf50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2b528adc42eb6754 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.049+0000 7f838affd700 1 -- 172.21.15.90:0/3377109283 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838c00fb60 con 0x55a23f7a40a0 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.049+0000 7f838affd700 1 -- 172.21.15.90:0/3377109283 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f838c00fcc0 con 0x55a23f7a40a0 2024-04-06T14:14:24.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.049+0000 7f838affd700 1 -- 172.21.15.90:0/3377109283 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838c011730 con 0x55a23f7a40a0 2024-04-06T14:14:24.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- 172.21.15.90:0/3377109283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 msgr2=0x55a23f7ad8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7ad8a0 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f838c00bc40 tx=0x7f838c00bf50 comp rx=0 tx=0).stop 2024-04-06T14:14:24.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- 172.21.15.90:0/3377109283 shutdown_connections 2024-04-06T14:14:24.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7b01d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a3780 0x55a23f7a3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 --2- 172.21.15.90:0/3377109283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7ad8a0 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- 172.21.15.90:0/3377109283 >> 172.21.15.90:0/3377109283 conn(0x55a23f7026a0 msgr2=0x55a23f79ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- 172.21.15.90:0/3377109283 shutdown_connections 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- 172.21.15.90:0/3377109283 wait complete. 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.050+0000 7f83b338db80 1 -- start start 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 Processor -- start 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7ad790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7adde0 0x55a23f7ab0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f690d80 con 0x55a23f7a40a0 2024-04-06T14:14:24.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f690a20 con 0x55a23f7adde0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f5786a0 con 0x55a23f7a3780 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.051+0000 7f83b338db80 10 client.0 _ll_register_callbacks cb 0x55a23f69a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39486/0 (socket says 172.21.15.90:39486) 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 -- 172.21.15.90:0/672161160 learned_addr learned my addr 172.21.15.90:0/672161160 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f83a4f75700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7ad790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 -- 172.21.15.90:0/672161160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 msgr2=0x55a23f7ad790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7ad790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 -- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7adde0 msgr2=0x55a23f7ab0b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7adde0 0x55a23f7ab0b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 -- 172.21.15.90:0/672161160 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83a0114ab0 con 0x55a23f7a40a0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f839ffff700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f838c000c00 tx=0x7f838c01ad10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=99670eee9470c556 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f837ffff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:24.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f838c01a870 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f838c01a9d0 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.052+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a23f6b5380 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f838c005020 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 10 client.16932 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f838c00b8a0 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55a23f6b5380 con 0x55a23f7a40a0 2024-04-06T14:14:24.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f838c02a050 con 0x55a23f7a40a0 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 10 client.16932 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 10 client.16932 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 20 client.16932 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7df880 0x55a23f7e1ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 4 client.16932 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a23f6ed700 con 0x55a23f7df880 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.053+0000 7f83a4f75700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7df880 0x55a23f7e1ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.054+0000 7f837ffff700 10 client.16932 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.054+0000 7f83a4f75700 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7df880 0x55a23f7e1ca0 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bf6765c7bcfb9556 server_cookie=a5ff4e50a0e28c3b in_seq=0 out_seq=0 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.054+0000 7f837ffff700 1 -- 172.21.15.90:0/672161160 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a23f6ed700 con 0x55a23f7df880 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.054+0000 7f837ffff700 10 client.16932 handle_command_reply: tid=0 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.054+0000 7f83b338db80 1 client.16932 shutdown 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 msgr2=0x55a23f7aaa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 secure :-1 s=READY pgs=692 cs=0 l=1 rev1=1 crypto rx=0x7f838c000c00 tx=0x7f838c01ad10 comp rx=0 tx=0).stop 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 shutdown_connections 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7adde0 0x55a23f7ab0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7df880 0x55a23f7e1ca0 crc :-1 s=READY pgs=163 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7ad790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 --2- 172.21.15.90:0/672161160 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7aaa80 unknown :-1 s=CLOSED pgs=692 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 >> 172.21.15.90:0/672161160 conn(0x55a23f7026a0 msgr2=0x55a23f79dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.055+0000 7f837ffff700 0 client.16932 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:24.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.058+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 shutdown_connections 2024-04-06T14:14:24.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.059+0000 7f83b338db80 1 -- 172.21.15.90:0/672161160 wait complete. 2024-04-06T14:14:24.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.062+0000 7f83b338db80 20 client.16932 trim_cache size 0 max 16384 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 Processor -- start 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 -- start start 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a3780 0x55a23f7aaa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7c5a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f79f830 con 0x55a23f7a40a0 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f7dc520 con 0x55a23f7a3780 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83b338db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f7a51f0 con 0x55a23f7adde0 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83a5776700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83a4f75700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a3780 0x55a23f7aaa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83a5776700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51420/0 (socket says 172.21.15.90:51420) 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.085+0000 7f83a5776700 1 -- 172.21.15.90:0/3343908499 learned_addr learned my addr 172.21.15.90:0/3343908499 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 -- 172.21.15.90:0/3343908499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a3780 msgr2=0x55a23f7aaa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a3780 0x55a23f7aaa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 -- 172.21.15.90:0/3343908499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 msgr2=0x55a23f7c5a70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7c5a70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 -- 172.21.15.90:0/3343908499 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a23f522b60 con 0x55a23f7adde0 2024-04-06T14:14:24.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f83a5776700 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f8398009f60 tx=0x7f8398099060 comp rx=0 tx=0).ready entity=mon.1 client_cookie=92c85ec0f66a88e5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f838b7fe700 1 -- 172.21.15.90:0/3343908499 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f839809d280 con 0x55a23f7adde0 2024-04-06T14:14:24.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f838b7fe700 1 -- 172.21.15.90:0/3343908499 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f83980a18d0 con 0x55a23f7adde0 2024-04-06T14:14:24.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.086+0000 7f838b7fe700 1 -- 172.21.15.90:0/3343908499 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8398099910 con 0x55a23f7adde0 2024-04-06T14:14:24.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.087+0000 7f83b338db80 1 -- 172.21.15.90:0/3343908499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 msgr2=0x55a23f7a9fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.087+0000 7f83b338db80 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f8398009f60 tx=0x7f8398099060 comp rx=0 tx=0).stop 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 -- 172.21.15.90:0/3343908499 shutdown_connections 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7adde0 0x55a23f7a9fc0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a3780 0x55a23f7aaa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 --2- 172.21.15.90:0/3343908499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7a40a0 0x55a23f7c5a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 -- 172.21.15.90:0/3343908499 >> 172.21.15.90:0/3343908499 conn(0x55a23f7026a0 msgr2=0x55a23f79a890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 -- 172.21.15.90:0/3343908499 shutdown_connections 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 -- 172.21.15.90:0/3343908499 wait complete. 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 -- start start 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.090+0000 7f83b338db80 1 Processor -- start 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7c3b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7adde0 0x55a23f7c4070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f7c6050 con 0x55a23f7adde0 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f7c45b0 con 0x55a23f7a40a0 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a23f7c4720 con 0x55a23f7a3780 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83b338db80 10 client.0 _ll_register_callbacks cb 0x55a23f69a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51434/0 (socket says 172.21.15.90:51434) 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 -- 172.21.15.90:0/572077407 learned_addr learned my addr 172.21.15.90:0/572077407 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 -- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 msgr2=0x55a23f7c3b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7c3b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 -- 172.21.15.90:0/572077407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7adde0 msgr2=0x55a23f7c4070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7adde0 0x55a23f7c4070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.091+0000 7f83a4f75700 1 -- 172.21.15.90:0/572077407 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83a01aab30 con 0x55a23f7a3780 2024-04-06T14:14:24.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83a4f75700 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f8394009990 tx=0x7f8394009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f2281d9d86c3f28e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8394029040 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f838c00fcc0 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8394033c70 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8394003e90 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 10 client.26428 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a23f6baea0 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f838c00fcc0 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8394005540 con 0x55a23f7a3780 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 10 client.26428 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 10 client.26428 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 20 client.26428 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7d6010 0x55a23f7d8450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:24.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 4 client.26428 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:24.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.092+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55a23f543cf0 con 0x55a23f7d6010 2024-04-06T14:14:24.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.093+0000 7f839ffff700 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7d6010 0x55a23f7d8450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:24.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.095+0000 7f839ffff700 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7d6010 0x55a23f7d8450 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c810ea2f7cbd5df0 server_cookie=1540c9bd4bf6a0bb in_seq=0 out_seq=0 2024-04-06T14:14:24.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.096+0000 7f837ffff700 10 client.26428 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:24.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.100+0000 7f837ffff700 1 -- 172.21.15.90:0/572077407 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55a23f543cf0 con 0x55a23f7d6010 2024-04-06T14:14:24.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.100+0000 7f837ffff700 10 client.26428 handle_command_reply: tid=0 2024-04-06T14:14:24.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.100+0000 7f83b338db80 1 client.26428 shutdown 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 msgr2=0x55a23f7c5740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f8394009990 tx=0x7f8394009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 shutdown_connections 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a23f7adde0 0x55a23f7c4070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a23f7d6010 0x55a23f7d8450 crc :-1 s=READY pgs=164 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a23f7a3780 0x55a23f7c5740 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 --2- 172.21.15.90:0/572077407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a23f7a40a0 0x55a23f7c3b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 >> 172.21.15.90:0/572077407 conn(0x55a23f7026a0 msgr2=0x55a23f7a5da0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f837ffff700 0 client.26428 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 shutdown_connections 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 1 -- 172.21.15.90:0/572077407 wait complete. 2024-04-06T14:14:24.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.101+0000 7f83b338db80 20 client.26428 trim_cache size 0 max 16384 2024-04-06T14:14:24.103 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 msgr2=0x7f83900a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 0x7f83900a3850 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8394000c00 tx=0x7f839400b580 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 msgr2=0x7f83a019de00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f839800b9e0 tx=0x7f839800bda0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 shutdown_connections 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f83900a1390 0x7f83900a3850 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f83a019d950 0x7f83a019de00 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f83a0073a70 0x7f83a01a37d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 --2- 172.21.15.90:0/859322660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f83a0111d10 0x7f83a01a3d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 >> 172.21.15.90:0/859322660 conn(0x7f83a00fc7f0 msgr2=0x7f83a00fe4e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 shutdown_connections 2024-04-06T14:14:24.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:24.102+0000 7f83a71d9700 1 -- 172.21.15.90:0/859322660 wait complete. 2024-04-06T14:14:24.146 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:23 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/614811401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:24.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:23 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/614811401' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:25.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:24 smithi090 ceph-mon[43420]: pgmap v279: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:25.122 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:14:25.122 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:14:25.122 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:14:25.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:24 smithi134 ceph-mon[47549]: pgmap v279: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:25.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:24 smithi105 ceph-mon[46969]: pgmap v279: 97 pgs: 97 active+clean; 6.8 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 16 MiB/s wr, 709 op/s 2024-04-06T14:14:25.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 -- 172.21.15.90:0/3325184046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 msgr2=0x7f981010bba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 --2- 172.21.15.90:0/3325184046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f981010bba0 secure :-1 s=READY pgs=693 cs=0 l=1 rev1=1 crypto rx=0x7f9804009990 tx=0x7f9804009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 -- 172.21.15.90:0/3325184046 shutdown_connections 2024-04-06T14:14:25.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 --2- 172.21.15.90:0/3325184046 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101185e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 --2- 172.21.15.90:0/3325184046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f9810111b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 --2- 172.21.15.90:0/3325184046 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f981010bba0 unknown :-1 s=CLOSED pgs=693 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.275+0000 7f9814c80700 1 -- 172.21.15.90:0/3325184046 >> 172.21.15.90:0/3325184046 conn(0x7f981006bd10 msgr2=0x7f9810109dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 -- 172.21.15.90:0/3325184046 shutdown_connections 2024-04-06T14:14:25.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 -- 172.21.15.90:0/3325184046 wait complete. 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 Processor -- start 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 -- start start 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101a7db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f98101a19a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.276+0000 7f9814c80700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98101a2000 con 0x7f981010b720 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f9814c80700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98101a2170 con 0x7f9810111720 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f9814c80700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98101a2330 con 0x7f981010c0e0 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39524/0 (socket says 172.21.15.90:39524) 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 -- 172.21.15.90:0/1985141507 learned_addr learned my addr 172.21.15.90:0/1985141507 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f98101a19a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980dd9b700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101a7db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 -- 172.21.15.90:0/1985141507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 msgr2=0x7f98101a7db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101a7db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 -- 172.21.15.90:0/1985141507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 msgr2=0x7f98101a19a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f98101a19a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 -- 172.21.15.90:0/1985141507 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98040095f0 con 0x7f981010b720 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f98101a19a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:25.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980dd9b700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101a7db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f980e59c700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f980000e9a0 tx=0x7f980000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f97ff7fe700 1 -- 172.21.15.90:0/1985141507 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9800009790 con 0x7f981010b720 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f97ff7fe700 1 -- 172.21.15.90:0/1985141507 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f980000cbd0 con 0x7f981010b720 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98101a2610 con 0x7f981010b720 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f97ff7fe700 1 -- 172.21.15.90:0/1985141507 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9800018680 con 0x7f981010b720 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.277+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f98101ae860 con 0x7f981010b720 2024-04-06T14:14:25.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.280+0000 7f97ff7fe700 1 -- 172.21.15.90:0/1985141507 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f980000cd40 con 0x7f981010b720 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.280+0000 7f97ff7fe700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 0x7f97f80a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.280+0000 7f97ff7fe700 1 -- 172.21.15.90:0/1985141507 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9800014070 con 0x7f981010b720 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.281+0000 7f980dd9b700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 0x7f97f80a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 Processor -- start 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f980dd9b700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 0x7f97f80a3750 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f980400b2e0 tx=0x7f9804005e80 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 -- start start 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be642dbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6437920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be643a250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be6385a10 con 0x560be642d800 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be63833d0 con 0x560be6437e60 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.282+0000 7f9820e34b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be637ecc0 con 0x560be642e120 2024-04-06T14:14:25.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be642dbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be642dbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39534/0 (socket says 172.21.15.90:39534) 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980e59c700 1 -- 172.21.15.90:0/1545515537 learned_addr learned my addr 172.21.15.90:0/1545515537 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be643a250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980dd9b700 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6437920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 -- 172.21.15.90:0/1545515537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 msgr2=0x560be6437920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6437920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 -- 172.21.15.90:0/1545515537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 msgr2=0x560be642dbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be642dbe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 -- 172.21.15.90:0/1545515537 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560be6200430 con 0x560be6437e60 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f980ed9d700 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be643a250 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f980800a5d0 tx=0x7f9808007820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=88c4596a8285b01e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f97fcff9700 1 -- 172.21.15.90:0/1545515537 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9808018070 con 0x560be6437e60 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f97fcff9700 1 -- 172.21.15.90:0/1545515537 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9808010040 con 0x560be6437e60 2024-04-06T14:14:25.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.283+0000 7f97fcff9700 1 -- 172.21.15.90:0/1545515537 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f980800f050 con 0x560be6437e60 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.284+0000 7f9820e34b80 1 -- 172.21.15.90:0/1545515537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 msgr2=0x560be643a250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.284+0000 7f9820e34b80 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be643a250 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f980800a5d0 tx=0x7f9808007820 comp rx=0 tx=0).stop 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 -- 172.21.15.90:0/1545515537 shutdown_connections 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be643a250 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be642dbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 --2- 172.21.15.90:0/1545515537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6437920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 -- 172.21.15.90:0/1545515537 >> 172.21.15.90:0/1545515537 conn(0x560be638c720 msgr2=0x560be6428b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 -- 172.21.15.90:0/1545515537 shutdown_connections 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 -- 172.21.15.90:0/1545515537 wait complete. 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 -- start start 2024-04-06T14:14:25.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.285+0000 7f9820e34b80 1 Processor -- start 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be6437890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6434bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be6435330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be631ae00 con 0x560be642d800 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be631aaa0 con 0x560be6437e60 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be6202720 con 0x560be642e120 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f9820e34b80 10 client.0 _ll_register_callbacks cb 0x560be63244f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be6437890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980e59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be6437890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39548/0 (socket says 172.21.15.90:39548) 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be6435330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980e59c700 1 -- 172.21.15.90:0/353552251 learned_addr learned my addr 172.21.15.90:0/353552251 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980dd9b700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6434bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 -- 172.21.15.90:0/353552251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 msgr2=0x560be6434bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6434bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 -- 172.21.15.90:0/353552251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 msgr2=0x560be6437890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be6437890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.286+0000 7f980ed9d700 1 -- 172.21.15.90:0/353552251 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9808007e30 con 0x560be6437e60 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f980ed9d700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be6435330 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f980800abc0 tx=0x7f9808003ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cc13d0722167f8c6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f97f5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:25.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9808018070 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9808003f80 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560be633f400 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9808007cd0 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f9820e34b80 10 client.26642 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.287+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560be61acbe0 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560be633f400 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9808014400 con 0x560be6437e60 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 10 client.26642 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:25.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 10 client.26642 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 20 client.26642 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be6469a70 0x560be646be90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 4 client.26642 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560be6377780 con 0x560be6469a70 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f980e59c700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be6469a70 0x560be646be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.288+0000 7f97f5ffb700 10 client.26642 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.289+0000 7f980e59c700 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be6469a70 0x560be646be90 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e9e2e4f136c142b5 server_cookie=444792025c688736 in_seq=0 out_seq=0 2024-04-06T14:14:25.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.289+0000 7f97f5ffb700 1 -- 172.21.15.90:0/353552251 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560be6377780 con 0x560be6469a70 2024-04-06T14:14:25.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.289+0000 7f97f5ffb700 10 client.26642 handle_command_reply: tid=0 2024-04-06T14:14:25.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.289+0000 7f9820e34b80 1 client.26642 shutdown 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 msgr2=0x560be6435330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be6435330 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f980800abc0 tx=0x7f9808003ce0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 shutdown_connections 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be6437e60 0x560be6435330 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642d800 0x560be6437890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be6469a70 0x560be646be90 crc :-1 s=READY pgs=165 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 --2- 172.21.15.90:0/353552251 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be642e120 0x560be6434bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 >> 172.21.15.90:0/353552251 conn(0x560be638c720 msgr2=0x560be6427ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f97f5ffb700 0 client.26642 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 shutdown_connections 2024-04-06T14:14:25.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 1 -- 172.21.15.90:0/353552251 wait complete. 2024-04-06T14:14:25.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.290+0000 7f9820e34b80 20 client.26642 trim_cache size 0 max 16384 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 Processor -- start 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 -- start start 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be6460ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be6430400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be642ea90 con 0x560be642e120 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be64296d0 con 0x560be642d800 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f9820e34b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be642f6d0 con 0x560be6437e60 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f980dd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f980dd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39552/0 (socket says 172.21.15.90:39552) 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f980e59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be6460ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.310+0000 7f980dd9b700 1 -- 172.21.15.90:0/3690346435 learned_addr learned my addr 172.21.15.90:0/3690346435 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 -- 172.21.15.90:0/3690346435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 msgr2=0x560be6430400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be6430400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 -- 172.21.15.90:0/3690346435 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 msgr2=0x560be642fec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 -- 172.21.15.90:0/3690346435 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560be6470ac0 con 0x560be642d800 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980dd9b700 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f980e59c700 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be6460ac0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f98000cac40 tx=0x7f980001a040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3739f20f9d9ea2ed server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f97fd7fa700 1 -- 172.21.15.90:0/3690346435 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f980009de30 con 0x560be642d800 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f97fd7fa700 1 -- 172.21.15.90:0/3690346435 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f98000a2480 con 0x560be642d800 2024-04-06T14:14:25.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.311+0000 7f97fd7fa700 1 -- 172.21.15.90:0/3690346435 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9800099750 con 0x560be642d800 2024-04-06T14:14:25.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.312+0000 7f9820e34b80 1 -- 172.21.15.90:0/3690346435 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 msgr2=0x560be6460ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.312+0000 7f9820e34b80 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be6460ac0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f98000cac40 tx=0x7f980001a040 comp rx=0 tx=0).stop 2024-04-06T14:14:25.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- 172.21.15.90:0/3690346435 shutdown_connections 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be6430400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be6460ac0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- 172.21.15.90:0/3690346435 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be642fec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- 172.21.15.90:0/3690346435 >> 172.21.15.90:0/3690346435 conn(0x560be638c720 msgr2=0x560be6434130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- 172.21.15.90:0/3690346435 shutdown_connections 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- 172.21.15.90:0/3690346435 wait complete. 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- start start 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 Processor -- start 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be6460210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be64b6a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be64b6f70 con 0x560be642e120 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be64b70e0 con 0x560be642d800 2024-04-06T14:14:25.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560be64b72a0 con 0x560be6437e60 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f9820e34b80 10 client.0 _ll_register_callbacks cb 0x560be63244f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.313+0000 7f980e59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980dd9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be6460210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54156/0 (socket says 172.21.15.90:54156) 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 -- 172.21.15.90:0/2114465374 learned_addr learned my addr 172.21.15.90:0/2114465374 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980ed9d700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be64b6a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 -- 172.21.15.90:0/2114465374 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 msgr2=0x560be64b6a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be64b6a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 -- 172.21.15.90:0/2114465374 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 msgr2=0x560be6460210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be6460210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 -- 172.21.15.90:0/2114465374 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9810118ee0 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f980e59c700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f9800096500 tx=0x7f980009a780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3e1cbf3aaa2d485 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f97f5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9800098460 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f98000b8280 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9808010040 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f98000994f0 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f9820e34b80 10 client.26648 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.314+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f98101ae860 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9808010040 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f98000b2bb0 con 0x560be642d800 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 10 client.26648 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 10 client.26648 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:25.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 20 client.26648 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be64bf9d0 0x560be64c1dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 4 client.26648 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x560be61cdd70 con 0x560be64bf9d0 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f980dd9b700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be64bf9d0 0x560be64c1dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f97f5ffb700 10 client.26648 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.315+0000 7f980dd9b700 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be64bf9d0 0x560be64c1dc0 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=27e96da7fc9756cd server_cookie=4b391d6b7891d755 in_seq=0 out_seq=0 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.316+0000 7f97f5ffb700 1 -- 172.21.15.90:0/2114465374 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x560be61cdd70 con 0x560be64bf9d0 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.316+0000 7f97f5ffb700 10 client.26648 handle_command_reply: tid=0 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.316+0000 7f9820e34b80 1 client.26648 shutdown 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 msgr2=0x560be645fcd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f9800096500 tx=0x7f980009a780 comp rx=0 tx=0).stop 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 shutdown_connections 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560be64bf9d0 0x560be64c1dc0 crc :-1 s=READY pgs=166 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560be6437e60 0x560be64b6a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f97f5ffb700 0 client.26648 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560be642d800 0x560be645fcd0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 --2- 172.21.15.90:0/2114465374 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560be642e120 0x560be6460210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 >> 172.21.15.90:0/2114465374 conn(0x560be638c720 msgr2=0x560be6433480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 shutdown_connections 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.317+0000 7f9820e34b80 1 -- 172.21.15.90:0/2114465374 wait complete. 2024-04-06T14:14:25.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.319+0000 7f9820e34b80 20 client.26648 trim_cache size 0 max 16384 2024-04-06T14:14:25.324 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:25.324 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:14:25.324 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "eb9ac2ad-1734-42eb-95c6-489d6927cbde", 2024-04-06T14:14:25.324 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:14:25.324 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.320+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 msgr2=0x7f97f80a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.320+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 0x7f97f80a3750 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f980400b2e0 tx=0x7f9804005e80 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.320+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 msgr2=0x7f98101a7870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.320+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 secure :-1 s=READY pgs=694 cs=0 l=1 rev1=1 crypto rx=0x7f980000e9a0 tx=0x7f980000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 shutdown_connections 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f97f80a1290 0x7f97f80a3750 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9810111720 0x7f98101a19a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f981010b720 0x7f98101a7870 unknown :-1 s=CLOSED pgs=694 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 --2- 172.21.15.90:0/1985141507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f981010c0e0 0x7f98101a7db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 >> 172.21.15.90:0/1985141507 conn(0x7f981006bd10 msgr2=0x7f981010e610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 shutdown_connections 2024-04-06T14:14:25.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.324+0000 7f9814c80700 1 -- 172.21.15.90:0/1985141507 wait complete. 2024-04-06T14:14:25.338 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:14:25.338 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:14:25.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.500+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2747295009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac068aa0 msgr2=0x7f65ac105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.500+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2747295009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac068aa0 0x7f65ac105970 secure :-1 s=READY pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f659c0099c0 tx=0x7f659c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2747295009 shutdown_connections 2024-04-06T14:14:25.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2747295009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac1060a0 0x7f65ac10dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2747295009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac0680f0 0x7f65ac0684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2747295009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac068aa0 0x7f65ac105970 secure :-1 s=CLOSED pgs=695 cs=0 l=1 rev1=1 crypto rx=0x7f659c0099c0 tx=0x7f659c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2747295009 >> 172.21.15.90:0/2747295009 conn(0x7f65ac076d50 msgr2=0x7f65ac077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.501+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2747295009 shutdown_connections 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.502+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2747295009 wait complete. 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 Processor -- start 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 -- start start 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 0x7f65ac197440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 0x7f65ac19c1a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ac19c7a0 con 0x7f65ac1060a0 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ac19c910 con 0x7f65ac19bd20 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b3e9a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65ac19cad0 con 0x7f65ac0680f0 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b1435700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b1435700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39584/0 (socket says 172.21.15.90:39584) 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b1435700 1 -- 172.21.15.90:0/2812477769 learned_addr learned my addr 172.21.15.90:0/2812477769 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b2437700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 0x7f65ac19c1a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.503+0000 7f65b1c36700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 0x7f65ac197440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 -- 172.21.15.90:0/2812477769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 msgr2=0x7f65ac197440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 0x7f65ac197440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 -- 172.21.15.90:0/2812477769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 msgr2=0x7f65ac19c1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 0x7f65ac19c1a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 -- 172.21.15.90:0/2812477769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f659c0095f0 con 0x7f65ac1060a0 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1c36700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 0x7f65ac197440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b2437700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 0x7f65ac19c1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b1435700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f659c0098e0 tx=0x7f659c00f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65a2ffd700 1 -- 172.21.15.90:0/2812477769 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f659c01c070 con 0x7f65ac1060a0 2024-04-06T14:14:25.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65a2ffd700 1 -- 172.21.15.90:0/2812477769 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f659c00fe30 con 0x7f65ac1060a0 2024-04-06T14:14:25.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65a2ffd700 1 -- 172.21.15.90:0/2812477769 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f659c022c60 con 0x7f65ac1060a0 2024-04-06T14:14:25.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65ac1a3fd0 con 0x7f65ac1060a0 2024-04-06T14:14:25.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.504+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65ac1a4340 con 0x7f65ac1060a0 2024-04-06T14:14:25.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 Processor -- start 2024-04-06T14:14:25.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 -- start start 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e7db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e8a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33dd5990 con 0x55ae33e7e0a0 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33dd3350 con 0x55ae33e87de0 2024-04-06T14:14:25.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65c004eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33dcec40 con 0x55ae33e7d780 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65b1c36700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65b1c36700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39598/0 (socket says 172.21.15.90:39598) 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65b1c36700 1 -- 172.21.15.90:0/481216447 learned_addr learned my addr 172.21.15.90:0/481216447 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.508+0000 7f65b2437700 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e7db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 -- 172.21.15.90:0/481216447 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 msgr2=0x55ae33e7db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e7db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 -- 172.21.15.90:0/481216447 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 msgr2=0x55ae33e8a1d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e8a1d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 -- 172.21.15.90:0/481216447 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ae33c503b0 con 0x55ae33e7e0a0 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65b1c36700 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f65a8009cc0 tx=0x7f65a8010370 comp rx=0 tx=0).ready entity=mon.0 client_cookie=511669176842c111 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f6597fff700 1 -- 172.21.15.90:0/481216447 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a8010bc0 con 0x55ae33e7e0a0 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f6597fff700 1 -- 172.21.15.90:0/481216447 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65a80044f0 con 0x55ae33e7e0a0 2024-04-06T14:14:25.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f6597fff700 1 -- 172.21.15.90:0/481216447 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a8003e90 con 0x55ae33e7e0a0 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.509+0000 7f65a2ffd700 1 -- 172.21.15.90:0/2812477769 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f659c020070 con 0x7f65ac1060a0 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65a2ffd700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 0x7f65980a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- 172.21.15.90:0/481216447 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 msgr2=0x55ae33e878a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 secure :-1 s=READY pgs=697 cs=0 l=1 rev1=1 crypto rx=0x7f65a8009cc0 tx=0x7f65a8010370 comp rx=0 tx=0).stop 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65b2437700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 0x7f65980a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65a2ffd700 1 -- 172.21.15.90:0/2812477769 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f659c0cd5d0 con 0x7f65ac1060a0 2024-04-06T14:14:25.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- 172.21.15.90:0/481216447 shutdown_connections 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e8a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e7db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 --2- 172.21.15.90:0/481216447 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7e0a0 0x55ae33e878a0 unknown :-1 s=CLOSED pgs=697 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- 172.21.15.90:0/481216447 >> 172.21.15.90:0/481216447 conn(0x55ae33ddc6a0 msgr2=0x55ae33e78a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- 172.21.15.90:0/481216447 shutdown_connections 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- 172.21.15.90:0/481216447 wait complete. 2024-04-06T14:14:25.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65b2437700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 0x7f65980a37d0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f65a4003fd0 tx=0x7f65a400b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.510+0000 7f65c004eb80 1 -- start start 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 Processor -- start 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7d780 0x55ae33e84c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e851b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33d6ad80 con 0x55ae33e87de0 2024-04-06T14:14:25.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33d6aa20 con 0x55ae33e7d780 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33c526a0 con 0x55ae33e7e0a0 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65c004eb80 10 client.0 _ll_register_callbacks cb 0x55ae33d74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39608/0 (socket says 172.21.15.90:39608) 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 -- 172.21.15.90:0/1986274015 learned_addr learned my addr 172.21.15.90:0/1986274015 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7d780 0x55ae33e84c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b1435700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e851b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 -- 172.21.15.90:0/1986274015 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 msgr2=0x55ae33e851b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e851b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 -- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7d780 msgr2=0x55ae33e84c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7d780 0x55ae33e84c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.511+0000 7f65b2437700 1 -- 172.21.15.90:0/1986274015 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65a8009920 con 0x55ae33e87de0 2024-04-06T14:14:25.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65b2437700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f65a4008ac0 tx=0x7f65a4004b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=619c36cca73428fb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a4004870 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f65a400e900 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ae33d8f380 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f65a400eac0 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 10 client.16962 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ae33bfcb60 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ae33d8f380 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f65a4016400 con 0x55ae33e87de0 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 10 client.16962 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 10 client.16962 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:25.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 20 client.16962 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.512+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33e97a40 0x55ae33e99e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.513+0000 7f65c004eb80 4 client.16962 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.513+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ae33dc7700 con 0x55ae33e97a40 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.513+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33e97a40 0x55ae33e99e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.514+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33e97a40 0x55ae33e99e60 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=15c118f01998362 server_cookie=27134771a92266b0 in_seq=0 out_seq=0 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.514+0000 7f65957fa700 10 client.16962 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.514+0000 7f65957fa700 1 -- 172.21.15.90:0/1986274015 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ae33dc7700 con 0x55ae33e97a40 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65957fa700 10 client.16962 handle_command_reply: tid=0 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 client.16962 shutdown 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 msgr2=0x55ae33e856f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 secure :-1 s=READY pgs=698 cs=0 l=1 rev1=1 crypto rx=0x7f65a4008ac0 tx=0x7f65a4004b40 comp rx=0 tx=0).stop 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 shutdown_connections 2024-04-06T14:14:25.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e856f0 unknown :-1 s=CLOSED pgs=698 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7d780 0x55ae33e84c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33e97a40 0x55ae33e99e60 crc :-1 s=READY pgs=167 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1986274015 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e851b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65957fa700 0 client.16962 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 >> 172.21.15.90:0/1986274015 conn(0x55ae33ddc6a0 msgr2=0x55ae33e77c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.515+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 shutdown_connections 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.516+0000 7f65c004eb80 1 -- 172.21.15.90:0/1986274015 wait complete. 2024-04-06T14:14:25.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.516+0000 7f65c004eb80 20 client.16962 trim_cache size 0 max 16384 2024-04-06T14:14:25.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.538+0000 7f65c004eb80 1 Processor -- start 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.542+0000 7f65c004eb80 1 -- start start 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 0x55ae33e7f650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e854e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e783a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33e7def0 con 0x55ae33e7d780 2024-04-06T14:14:25.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33e70330 con 0x55ae33e87de0 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.543+0000 7f65c004eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33e87ba0 con 0x55ae33e7e0a0 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.547+0000 7f65b2437700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e783a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.547+0000 7f65b2437700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e783a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:54178/0 (socket says 172.21.15.90:54178) 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.547+0000 7f65b2437700 1 -- 172.21.15.90:0/1059187801 learned_addr learned my addr 172.21.15.90:0/1059187801 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.547+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 0x55ae33e7f650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 -- 172.21.15.90:0/1059187801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 msgr2=0x55ae33e854e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e854e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 -- 172.21.15.90:0/1059187801 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 msgr2=0x55ae33e783a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e783a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 -- 172.21.15.90:0/1059187801 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ae33d94ea0 con 0x55ae33e7d780 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.548+0000 7f65b1c36700 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 0x55ae33e7f650 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f65a8000c00 tx=0x7f65a8003a70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c1414279c1e4ba57 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.550+0000 7f65a0ff9700 1 -- 172.21.15.90:0/1059187801 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a8028a10 con 0x55ae33e7d780 2024-04-06T14:14:25.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.551+0000 7f65a0ff9700 1 -- 172.21.15.90:0/1059187801 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65a801f040 con 0x55ae33e7d780 2024-04-06T14:14:25.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.551+0000 7f65a0ff9700 1 -- 172.21.15.90:0/1059187801 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a8028360 con 0x55ae33e7d780 2024-04-06T14:14:25.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.551+0000 7f65c004eb80 1 -- 172.21.15.90:0/1059187801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 msgr2=0x55ae33e7f650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.551+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 0x55ae33e7f650 secure :-1 s=READY pgs=699 cs=0 l=1 rev1=1 crypto rx=0x7f65a8000c00 tx=0x7f65a8003a70 comp rx=0 tx=0).stop 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 -- 172.21.15.90:0/1059187801 shutdown_connections 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e87de0 0x55ae33e783a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e7d780 0x55ae33e7f650 unknown :-1 s=CLOSED pgs=699 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 --2- 172.21.15.90:0/1059187801 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7e0a0 0x55ae33e854e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 -- 172.21.15.90:0/1059187801 >> 172.21.15.90:0/1059187801 conn(0x55ae33ddc6a0 msgr2=0x55ae33e83540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 -- 172.21.15.90:0/1059187801 shutdown_connections 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 -- 172.21.15.90:0/1059187801 wait complete. 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 -- start start 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.553+0000 7f65c004eb80 1 Processor -- start 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e8ea90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7e0a0 0x55ae33e8ce80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33e8dad0 con 0x55ae33e87de0 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33e8dc40 con 0x55ae33e7e0a0 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae33eee060 con 0x55ae33e7d780 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65c004eb80 10 client.0 _ll_register_callbacks cb 0x55ae33d74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65b2437700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.554+0000 7f65b2437700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:39624/0 (socket says 172.21.15.90:39624) 2024-04-06T14:14:25.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 -- 172.21.15.90:0/179530748 learned_addr learned my addr 172.21.15.90:0/179530748 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b1435700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7e0a0 0x55ae33e8ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b1c36700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e8ea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 -- 172.21.15.90:0/179530748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 msgr2=0x55ae33e8ea90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e8ea90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 -- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7e0a0 msgr2=0x55ae33e8ce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7e0a0 0x55ae33e8ce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 -- 172.21.15.90:0/179530748 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ae33c503b0 con 0x55ae33e87de0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65b2437700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f65a40047e0 tx=0x7f65a4004aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=caf07dd839ed4917 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65957fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.555+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f65a400e900 con 0x55ae33e87de0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.556+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f65a400ea60 con 0x55ae33e87de0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.556+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f65a80044f0 con 0x55ae33e87de0 2024-04-06T14:14:25.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.556+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f65a400ec20 con 0x55ae33e87de0 2024-04-06T14:14:25.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.557+0000 7f65c004eb80 10 client.16968 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:25.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.557+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ae33c54cd0 con 0x55ae33e87de0 2024-04-06T14:14:25.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f65a80044f0 con 0x55ae33e87de0 2024-04-06T14:14:25.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f65a402c440 con 0x55ae33e87de0 2024-04-06T14:14:25.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 10 client.16968 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 10 client.16968 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 20 client.16968 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33eef640 0x55ae33eefa50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 4 client.16968 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ae33c1dcf0 con 0x55ae33eef640 2024-04-06T14:14:25.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.558+0000 7f65b1c36700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33eef640 0x55ae33eefa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:25.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.560+0000 7f65957fa700 10 client.16968 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.560+0000 7f65b1c36700 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33eef640 0x55ae33eefa50 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1ce34bbf29774324 server_cookie=73b885f55fe51869 in_seq=0 out_seq=0 2024-04-06T14:14:25.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.560+0000 7f65957fa700 1 -- 172.21.15.90:0/179530748 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55ae33c1dcf0 con 0x55ae33eef640 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.560+0000 7f65957fa700 10 client.16968 handle_command_reply: tid=0 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.560+0000 7f65c004eb80 1 client.16968 shutdown 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 msgr2=0x55ae33e8d3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 secure :-1 s=READY pgs=700 cs=0 l=1 rev1=1 crypto rx=0x7f65a40047e0 tx=0x7f65a4004aa0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 shutdown_connections 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae33e87de0 0x55ae33e8d3c0 unknown :-1 s=CLOSED pgs=700 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae33eef640 0x55ae33eefa50 crc :-1 s=READY pgs=168 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae33e7d780 0x55ae33e8ea90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 --2- 172.21.15.90:0/179530748 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae33e7e0a0 0x55ae33e8ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65957fa700 0 client.16968 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 >> 172.21.15.90:0/179530748 conn(0x55ae33ddc6a0 msgr2=0x55ae33e7d000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.561+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 shutdown_connections 2024-04-06T14:14:25.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.562+0000 7f65c004eb80 1 -- 172.21.15.90:0/179530748 wait complete. 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.562+0000 7f65c004eb80 20 client.16968 trim_cache size 0 max 16384 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "eb9ac2ad-1734-42eb-95c6-489d6927cbde": { 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "eb9ac2ad-1734-42eb-95c6-489d6927cbde", 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:14:25.570 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 msgr2=0x7f65980a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 0x7f65980a37d0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f65a4003fd0 tx=0x7f65a400b040 comp rx=0 tx=0).stop 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 msgr2=0x7f65ac197980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 secure :-1 s=READY pgs=696 cs=0 l=1 rev1=1 crypto rx=0x7f659c0098e0 tx=0x7f659c00f690 comp rx=0 tx=0).stop 2024-04-06T14:14:25.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 shutdown_connections 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f65980a1310 0x7f65980a37d0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65ac19bd20 0x7f65ac19c1a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f65ac0680f0 0x7f65ac197440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 --2- 172.21.15.90:0/2812477769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f65ac1060a0 0x7f65ac197980 unknown :-1 s=CLOSED pgs=696 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.563+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 >> 172.21.15.90:0/2812477769 conn(0x7f65ac076d50 msgr2=0x7f65ac0fdb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.566+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 shutdown_connections 2024-04-06T14:14:25.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:25.566+0000 7f65b3e9a700 1 -- 172.21.15.90:0/2812477769 wait complete. 2024-04-06T14:14:25.582 INFO:tasks.cephfs.filesystem:scrub status for tag:eb9ac2ad-1734-42eb-95c6-489d6927cbde - {'path': '/', 'tag': 'eb9ac2ad-1734-42eb-95c6-489d6927cbde', 'options': 'recursive,force'} 2024-04-06T14:14:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:25 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:14:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:25 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:26.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:25 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:14:26.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:25 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:26.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:25 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:14:26.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:25 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:26.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:25 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:14:26.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:25 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:26.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:25 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:14:26.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:25 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:26.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:25 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:14:26.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:25 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:27.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:26 smithi090 ceph-mon[43420]: pgmap v280: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 161 KiB/s rd, 27 MiB/s wr, 1.94k op/s 2024-04-06T14:14:27.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:26 smithi134 ceph-mon[47549]: pgmap v280: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 161 KiB/s rd, 27 MiB/s wr, 1.94k op/s 2024-04-06T14:14:27.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:26 smithi105 ceph-mon[46969]: pgmap v280: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 161 KiB/s rd, 27 MiB/s wr, 1.94k op/s 2024-04-06T14:14:28.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 520.0 s, 1934.1 tps, lat 51.846 ms stddev 65.403 2024-04-06T14:14:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:28 smithi090 ceph-mon[43420]: pgmap v281: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 160 KiB/s rd, 27 MiB/s wr, 1.93k op/s 2024-04-06T14:14:29.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:28 smithi134 ceph-mon[47549]: pgmap v281: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 160 KiB/s rd, 27 MiB/s wr, 1.93k op/s 2024-04-06T14:14:29.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:28 smithi105 ceph-mon[46969]: pgmap v281: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 160 KiB/s rd, 27 MiB/s wr, 1.93k op/s 2024-04-06T14:14:31.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:30 smithi105 ceph-mon[46969]: pgmap v282: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 22 MiB/s wr, 1.62k op/s 2024-04-06T14:14:31.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:30 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:14:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:30 smithi090 ceph-mon[43420]: pgmap v282: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 22 MiB/s wr, 1.62k op/s 2024-04-06T14:14:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:30 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:14:31.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:30 smithi134 ceph-mon[47549]: pgmap v282: 97 pgs: 97 active+clean; 7.0 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 22 MiB/s wr, 1.62k op/s 2024-04-06T14:14:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:30 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:14:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 525.0 s, 2020.7 tps, lat 49.875 ms stddev 57.922 2024-04-06T14:14:33.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:32 smithi105 ceph-mon[46969]: pgmap v283: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 28 MiB/s wr, 1.86k op/s 2024-04-06T14:14:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:32 smithi090 ceph-mon[43420]: pgmap v283: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 28 MiB/s wr, 1.86k op/s 2024-04-06T14:14:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:32 smithi134 ceph-mon[47549]: pgmap v283: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 28 MiB/s wr, 1.86k op/s 2024-04-06T14:14:35.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:34 smithi105 ceph-mon[46969]: pgmap v284: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 21 MiB/s wr, 1.54k op/s 2024-04-06T14:14:35.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:34 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:14:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:34 smithi090 ceph-mon[43420]: pgmap v284: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 21 MiB/s wr, 1.54k op/s 2024-04-06T14:14:35.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:34 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:14:35.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:34 smithi134 ceph-mon[47549]: pgmap v284: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 21 MiB/s wr, 1.54k op/s 2024-04-06T14:14:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:34 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:14:36.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:14:36.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:14:36.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:14:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:14:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:14:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:14:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:14:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:14:36.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:14:37.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:36 smithi105 ceph-mon[46969]: pgmap v285: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 245 KiB/s rd, 24 MiB/s wr, 1.75k op/s 2024-04-06T14:14:37.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:36 smithi090 ceph-mon[43420]: pgmap v285: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 245 KiB/s rd, 24 MiB/s wr, 1.75k op/s 2024-04-06T14:14:37.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:36 smithi134 ceph-mon[47549]: pgmap v285: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 245 KiB/s rd, 24 MiB/s wr, 1.75k op/s 2024-04-06T14:14:38.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 530.0 s, 1629.8 tps, lat 61.358 ms stddev 103.254 2024-04-06T14:14:39.156 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:38 smithi134 ceph-mon[47549]: pgmap v286: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:38 smithi105 ceph-mon[46969]: pgmap v286: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:38 smithi090 ceph-mon[43420]: pgmap v286: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:41.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:40 smithi105 ceph-mon[46969]: pgmap v287: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:41.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:40 smithi090 ceph-mon[43420]: pgmap v287: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:41.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:40 smithi134 ceph-mon[47549]: pgmap v287: 97 pgs: 97 active+clean; 7.1 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 13 MiB/s wr, 782 op/s 2024-04-06T14:14:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 535.0 s, 2042.8 tps, lat 48.550 ms stddev 56.884 2024-04-06T14:14:43.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:42 smithi105 ceph-mon[46969]: pgmap v288: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 24 MiB/s wr, 1.68k op/s 2024-04-06T14:14:43.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:42 smithi090 ceph-mon[43420]: pgmap v288: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 24 MiB/s wr, 1.68k op/s 2024-04-06T14:14:43.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:42 smithi134 ceph-mon[47549]: pgmap v288: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 24 MiB/s wr, 1.68k op/s 2024-04-06T14:14:45.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:44 smithi105 ceph-mon[46969]: pgmap v289: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 19 MiB/s wr, 1.44k op/s 2024-04-06T14:14:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:44 smithi090 ceph-mon[43420]: pgmap v289: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 19 MiB/s wr, 1.44k op/s 2024-04-06T14:14:45.400 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:44 smithi134 ceph-mon[47549]: pgmap v289: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 19 MiB/s wr, 1.44k op/s 2024-04-06T14:14:47.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:46 smithi105 ceph-mon[46969]: pgmap v290: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:14:47.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:46 smithi090 ceph-mon[43420]: pgmap v290: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:14:47.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:46 smithi134 ceph-mon[47549]: pgmap v290: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:14:48.032 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 540.0 s, 2206.6 tps, lat 45.735 ms stddev 55.333 2024-04-06T14:14:49.040 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:48 smithi090 ceph-mon[43420]: pgmap v291: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 20 MiB/s wr, 1.50k op/s 2024-04-06T14:14:49.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:48 smithi134 ceph-mon[47549]: pgmap v291: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 20 MiB/s wr, 1.50k op/s 2024-04-06T14:14:49.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:48 smithi105 ceph-mon[46969]: pgmap v291: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 20 MiB/s wr, 1.50k op/s 2024-04-06T14:14:51.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:50 smithi105 ceph-mon[46969]: pgmap v292: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 15 MiB/s wr, 1.17k op/s 2024-04-06T14:14:51.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:50 smithi090 ceph-mon[43420]: pgmap v292: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 15 MiB/s wr, 1.17k op/s 2024-04-06T14:14:51.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:50 smithi134 ceph-mon[47549]: pgmap v292: 97 pgs: 97 active+clean; 7.2 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 15 MiB/s wr, 1.17k op/s 2024-04-06T14:14:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 545.0 s, 1936.3 tps, lat 51.595 ms stddev 61.703 2024-04-06T14:14:53.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:52 smithi105 ceph-mon[46969]: pgmap v293: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 20 MiB/s wr, 1.45k op/s 2024-04-06T14:14:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:52 smithi090 ceph-mon[43420]: pgmap v293: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 20 MiB/s wr, 1.45k op/s 2024-04-06T14:14:53.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:52 smithi134 ceph-mon[47549]: pgmap v293: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 20 MiB/s wr, 1.45k op/s 2024-04-06T14:14:55.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:54 smithi105 ceph-mon[46969]: pgmap v294: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 8.6 MiB/s wr, 555 op/s 2024-04-06T14:14:55.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:54 smithi090 ceph-mon[43420]: pgmap v294: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 8.6 MiB/s wr, 555 op/s 2024-04-06T14:14:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:54 smithi134 ceph-mon[47549]: pgmap v294: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 8.6 MiB/s wr, 555 op/s 2024-04-06T14:14:55.583 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:14:55.583 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:14:55.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 -- 172.21.15.90:0/3701248237 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 msgr2=0x7feda0108400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 --2- 172.21.15.90:0/3701248237 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0108400 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fed94009730 tx=0x7fed940075a0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 -- 172.21.15.90:0/3701248237 shutdown_connections 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 --2- 172.21.15.90:0/3701248237 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda010c6a0 0x7feda00790c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 --2- 172.21.15.90:0/3701248237 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0108400 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 --2- 172.21.15.90:0/3701248237 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda0108940 0x7feda010bf70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.770+0000 7feda5641700 1 -- 172.21.15.90:0/3701248237 >> 172.21.15.90:0/3701248237 conn(0x7feda006be00 msgr2=0x7feda006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.771+0000 7feda5641700 1 -- 172.21.15.90:0/3701248237 shutdown_connections 2024-04-06T14:14:55.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.771+0000 7feda5641700 1 -- 172.21.15.90:0/3701248237 wait complete. 2024-04-06T14:14:55.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 Processor -- start 2024-04-06T14:14:55.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 -- start start 2024-04-06T14:14:55.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0081c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 0x7feda0082180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda010c6a0 0x7feda00826c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feda01bef30 con 0x7feda010c6a0 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feda01bf0a0 con 0x7feda0108940 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.772+0000 7feda5641700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feda01bf260 con 0x7feda0108020 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0081c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 0x7feda0082180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda010c6a0 0x7feda00826c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0081c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57050/0 (socket says 172.21.15.90:57050) 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9effd700 1 -- 172.21.15.90:0/1234630952 learned_addr learned my addr 172.21.15.90:0/1234630952 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 -- 172.21.15.90:0/1234630952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 msgr2=0x7feda0081c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0081c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 -- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda010c6a0 msgr2=0x7feda00826c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda010c6a0 0x7feda00826c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 -- 172.21.15.90:0/1234630952 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed9400d040 con 0x7feda0108940 2024-04-06T14:14:55.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.773+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 0x7feda0082180 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fed9800bd70 tx=0x7fed9800be50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.774+0000 7fed87fff700 1 -- 172.21.15.90:0/1234630952 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed9800cd50 con 0x7feda0108940 2024-04-06T14:14:55.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.774+0000 7fed87fff700 1 -- 172.21.15.90:0/1234630952 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fed98014900 con 0x7feda0108940 2024-04-06T14:14:55.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.774+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feda01bf540 con 0x7feda0108940 2024-04-06T14:14:55.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.774+0000 7fed87fff700 1 -- 172.21.15.90:0/1234630952 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed98012820 con 0x7feda0108940 2024-04-06T14:14:55.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.774+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feda01bfaa0 con 0x7feda0108940 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.777+0000 7fed87fff700 1 -- 172.21.15.90:0/1234630952 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fed98012980 con 0x7feda0108940 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.778+0000 7fed87fff700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 0x7fed880a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.778+0000 7fed87fff700 1 -- 172.21.15.90:0/1234630952 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fed98006e40 con 0x7feda0108940 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.779+0000 7fed9effd700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 0x7fed880a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.779+0000 7fedb17f5b80 1 Processor -- start 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.780+0000 7fed9effd700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 0x7fed880a3810 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fed94007570 tx=0x7fed9400b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 -- start start 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9eda780 0x5596d9edab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9e32990 con 0x5596d9edb0a0 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9e30350 con 0x5596d9eda780 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9e2bc40 con 0x5596d9ee4de0 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47798/0 (socket says 172.21.15.90:47798) 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3508432800 learned_addr learned my addr 172.21.15.90:0/3508432800 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9f7fe700 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9effd700 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9eda780 0x5596d9edab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3508432800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 msgr2=0x5596d9ee71d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee71d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3508432800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9eda780 msgr2=0x5596d9edab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9eda780 0x5596d9edab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.782+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3508432800 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5596d9cad3b0 con 0x5596d9edb0a0 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fed98096c40 tx=0x7fed9809a3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c59cb299ceb3880f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fed857fa700 1 -- 172.21.15.90:0/3508432800 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed9809e570 con 0x5596d9edb0a0 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fed857fa700 1 -- 172.21.15.90:0/3508432800 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fed980a2bc0 con 0x5596d9edb0a0 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fed857fa700 1 -- 172.21.15.90:0/3508432800 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed980ba1c0 con 0x5596d9edb0a0 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3508432800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 msgr2=0x5596d9ee48a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.783+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 secure :-1 s=READY pgs=701 cs=0 l=1 rev1=1 crypto rx=0x7fed98096c40 tx=0x7fed9809a3b0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3508432800 shutdown_connections 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee71d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9eda780 0x5596d9edab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3508432800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9edb0a0 0x5596d9ee48a0 unknown :-1 s=CLOSED pgs=701 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3508432800 >> 172.21.15.90:0/3508432800 conn(0x5596d9e396a0 msgr2=0x5596d9ed5a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3508432800 shutdown_connections 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3508432800 wait complete. 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- start start 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 Processor -- start 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee2180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee26c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9dc7d80 con 0x5596d9eda780 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9dc7a20 con 0x5596d9edb0a0 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9caf6a0 con 0x5596d9ee4de0 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fedb17f5b80 10 client.0 _ll_register_callbacks cb 0x5596d9dd1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fed9effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fed9effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47804/0 (socket says 172.21.15.90:47804) 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.784+0000 7fed9effd700 1 -- 172.21.15.90:0/4245159861 learned_addr learned my addr 172.21.15.90:0/4245159861 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9f7fe700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee26c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee2180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 -- 172.21.15.90:0/4245159861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 msgr2=0x5596d9ee26c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee26c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 -- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 msgr2=0x5596d9ee2180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee2180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 -- 172.21.15.90:0/4245159861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feda01bfaa0 con 0x5596d9eda780 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee2180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed9effd700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fed94003ad0 tx=0x7fed940075a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=72abb64703eac364 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed82ff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed94004e70 con 0x5596d9eda780 2024-04-06T14:14:55.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fed94007610 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.785+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5596d9dec380 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fed940077d0 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 10 client.16977 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5596d9c59b60 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5596d9dec380 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fed9401bb40 con 0x5596d9eda780 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 10 client.16977 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 10 client.16977 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 20 client.16977 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9ef48d0 0x5596d9ef6cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 4 client.16977 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.788+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5596d9e24700 con 0x5596d9ef48d0 2024-04-06T14:14:55.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.789+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9ef48d0 0x5596d9ef6cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.789+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9ef48d0 0x5596d9ef6cf0 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=be747d2da9bd89d9 server_cookie=2d24a1c54b5f438c in_seq=0 out_seq=0 2024-04-06T14:14:55.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.789+0000 7fed82ff5700 10 client.16977 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:55.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.790+0000 7fed82ff5700 1 -- 172.21.15.90:0/4245159861 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5596d9e24700 con 0x5596d9ef48d0 2024-04-06T14:14:55.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.790+0000 7fed82ff5700 10 client.16977 handle_command_reply: tid=0 2024-04-06T14:14:55.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.790+0000 7fedb17f5b80 1 client.16977 shutdown 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 msgr2=0x5596d9ee1c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 secure :-1 s=READY pgs=702 cs=0 l=1 rev1=1 crypto rx=0x7fed94003ad0 tx=0x7fed940075a0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 shutdown_connections 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9ef48d0 0x5596d9ef6cf0 crc :-1 s=READY pgs=169 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9ee4de0 0x5596d9ee26c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9eda780 0x5596d9ee1c40 unknown :-1 s=CLOSED pgs=702 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/4245159861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee2180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 >> 172.21.15.90:0/4245159861 conn(0x5596d9e396a0 msgr2=0x5596d9ed4c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fed82ff5700 0 client.16977 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 shutdown_connections 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 1 -- 172.21.15.90:0/4245159861 wait complete. 2024-04-06T14:14:55.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.792+0000 7fedb17f5b80 20 client.16977 trim_cache size 0 max 16384 2024-04-06T14:14:55.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.814+0000 7fedb17f5b80 1 Processor -- start 2024-04-06T14:14:55.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 -- start start 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9ed5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ed5a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9edeee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9edc630 con 0x5596d9ee4de0 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9edba10 con 0x5596d9edb0a0 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9edaef0 con 0x5596d9eda780 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fed9effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9ed5520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fed9effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9ed5520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57070/0 (socket says 172.21.15.90:57070) 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fed9effd700 1 -- 172.21.15.90:0/3259678705 learned_addr learned my addr 172.21.15.90:0/3259678705 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ed5a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.815+0000 7fed9f7fe700 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9edeee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3259678705 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 msgr2=0x5596d9ed5520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9ed5520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3259678705 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 msgr2=0x5596d9edeee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9edeee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 -- 172.21.15.90:0/3259678705 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5596d9df1ea0 con 0x5596d9edb0a0 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ed5a60 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fed9800cfd0 tx=0x7fed980ba040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8e7e116cb35dec91 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed85ffb700 1 -- 172.21.15.90:0/3259678705 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed980b3130 con 0x5596d9edb0a0 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed85ffb700 1 -- 172.21.15.90:0/3259678705 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fed980b8a50 con 0x5596d9edb0a0 2024-04-06T14:14:55.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.816+0000 7fed85ffb700 1 -- 172.21.15.90:0/3259678705 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed98096e50 con 0x5596d9edb0a0 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3259678705 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 msgr2=0x5596d9ed5a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ed5a60 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fed9800cfd0 tx=0x7fed980ba040 comp rx=0 tx=0).stop 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3259678705 shutdown_connections 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9edeee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9ed5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/3259678705 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ed5a60 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3259678705 >> 172.21.15.90:0/3259678705 conn(0x5596d9e396a0 msgr2=0x5596d9ee0530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3259678705 shutdown_connections 2024-04-06T14:14:55.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.817+0000 7fedb17f5b80 1 -- 172.21.15.90:0/3259678705 wait complete. 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 -- start start 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 Processor -- start 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9eebb40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9eea5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9eead00 con 0x5596d9ee4de0 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9eeae70 con 0x5596d9edb0a0 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5596d9f48790 con 0x5596d9eda780 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fedb17f5b80 10 client.0 _ll_register_callbacks cb 0x5596d9dd1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fed9e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fed9e7fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34508/0 (socket says 172.21.15.90:34508) 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.818+0000 7fed9e7fc700 1 -- 172.21.15.90:0/260688155 learned_addr learned my addr 172.21.15.90:0/260688155 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:55.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9f7fe700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9eea5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 -- 172.21.15.90:0/260688155 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 msgr2=0x5596d9eebb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9eebb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 -- 172.21.15.90:0/260688155 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 msgr2=0x5596d9eea5f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9eea5f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 -- 172.21.15.90:0/260688155 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5596d9cad3b0 con 0x5596d9edb0a0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9f7fe700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9eea5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed9e7fc700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fed9800bf70 tx=0x7fed98004be0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b52a34c177350b74 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed82ff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed980b3130 con 0x5596d9edb0a0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fed980a2bc0 con 0x5596d9edb0a0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fed980ba0b0 con 0x5596d9edb0a0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fed98096e50 con 0x5596d9edb0a0 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fedb17f5b80 10 client.26681 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:55.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5596d9cb1cd0 con 0x5596d9edb0a0 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.819+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fed980a2bc0 con 0x5596d9edb0a0 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fed98012cc0 con 0x5596d9edb0a0 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 10 client.26681 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 10 client.26681 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 20 client.26681 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9f49a50 0x5596d9f4be40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 4 client.26681 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5596d9c7acf0 con 0x5596d9f49a50 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fed9effd700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9f49a50 0x5596d9f4be40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.820+0000 7fed82ff5700 10 client.26681 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fed9effd700 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9f49a50 0x5596d9f4be40 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b835b452bacf6323 server_cookie=e7646978c502e5d1 in_seq=0 out_seq=0 2024-04-06T14:14:55.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fed82ff5700 1 -- 172.21.15.90:0/260688155 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x5596d9c7acf0 con 0x5596d9f49a50 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fed82ff5700 10 client.26681 handle_command_reply: tid=0 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 client.26681 shutdown 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 msgr2=0x5596d9ee9fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fed9800bf70 tx=0x7fed98004be0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 shutdown_connections 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5596d9ee4de0 0x5596d9eea5f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5596d9f49a50 0x5596d9f4be40 crc :-1 s=READY pgs=170 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5596d9eda780 0x5596d9eebb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fed82ff5700 0 client.26681 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 --2- 172.21.15.90:0/260688155 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5596d9edb0a0 0x5596d9ee9fe0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 >> 172.21.15.90:0/260688155 conn(0x5596d9e396a0 msgr2=0x5596d9ee0490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 shutdown_connections 2024-04-06T14:14:55.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 1 -- 172.21.15.90:0/260688155 wait complete. 2024-04-06T14:14:55.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.821+0000 7fedb17f5b80 20 client.26681 trim_cache size 0 max 16384 2024-04-06T14:14:55.827 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:55.827 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:14:55.827 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:14:55.827 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 msgr2=0x7fed880a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 0x7fed880a3810 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fed94007570 tx=0x7fed9400b040 comp rx=0 tx=0).stop 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 msgr2=0x7feda0082180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 0x7feda0082180 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fed9800bd70 tx=0x7fed9800be50 comp rx=0 tx=0).stop 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 shutdown_connections 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fed880a1350 0x7fed880a3810 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7feda010c6a0 0x7feda00826c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7feda0108020 0x7feda0081c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 --2- 172.21.15.90:0/1234630952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feda0108940 0x7feda0082180 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:55.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.822+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 >> 172.21.15.90:0/1234630952 conn(0x7feda006be00 msgr2=0x7feda0070aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:55.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.823+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 shutdown_connections 2024-04-06T14:14:55.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:55.823+0000 7feda5641700 1 -- 172.21.15.90:0/1234630952 wait complete. 2024-04-06T14:14:55.843 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:14:55.843 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:14:56.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 -- 172.21.15.90:0/1026019219 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 msgr2=0x7f3c54101270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 --2- 172.21.15.90:0/1026019219 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54101270 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f3c440098f0 tx=0x7f3c44009c00 comp rx=0 tx=0).stop 2024-04-06T14:14:56.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 -- 172.21.15.90:0/1026019219 shutdown_connections 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 --2- 172.21.15.90:0/1026019219 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c54105fb0 0x7f3c5410db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 --2- 172.21.15.90:0/1026019219 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54101270 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 --2- 172.21.15.90:0/1026019219 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54101840 0x7f3c54105880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 -- 172.21.15.90:0/1026019219 >> 172.21.15.90:0/1026019219 conn(0x7f3c540fc750 msgr2=0x7f3c540feb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 -- 172.21.15.90:0/1026019219 shutdown_connections 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.059+0000 7f3c5a858700 1 -- 172.21.15.90:0/1026019219 wait complete. 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.060+0000 7f3c5a858700 1 Processor -- start 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.060+0000 7f3c5a858700 1 -- start start 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.060+0000 7f3c5a858700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54197270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54105fb0 0x7f3c541977b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5419c580 con 0x7f3c5419bb00 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5419c6f0 con 0x7f3c54105fb0 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c5419c8b0 con 0x7f3c54100e90 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47836/0 (socket says 172.21.15.90:47836) 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c53fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54197270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 -- 172.21.15.90:0/513533807 learned_addr learned my addr 172.21.15.90:0/513533807 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c537fe700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54105fb0 0x7f3c541977b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 -- 172.21.15.90:0/513533807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 msgr2=0x7f3c54197270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54197270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 -- 172.21.15.90:0/513533807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54105fb0 msgr2=0x7f3c541977b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54105fb0 0x7f3c541977b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 -- 172.21.15.90:0/513533807 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c44009520 con 0x7f3c5419bb00 2024-04-06T14:14:56.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c53fff700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54197270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c58df5700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c00c110 tx=0x7f3c4c00c420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c4c017070 con 0x7f3c5419bb00 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c541a3db0 con 0x7f3c5419bb00 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c4c00e050 con 0x7f3c5419bb00 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.061+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c4c0078f0 con 0x7f3c5419bb00 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.062+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c541a4290 con 0x7f3c5419bb00 2024-04-06T14:14:56.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.064+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c54109220 con 0x7f3c5419bb00 2024-04-06T14:14:56.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.064+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3c4c007a50 con 0x7f3c5419bb00 2024-04-06T14:14:56.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.064+0000 7f3c517fa700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 0x7f3c3c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.065+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3c4c0cc6a0 con 0x7f3c5419bb00 2024-04-06T14:14:56.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.066+0000 7f3c53fff700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 0x7f3c3c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.066+0000 7f3c53fff700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 0x7f3c3c0a3750 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f3c440098c0 tx=0x7f3c44019040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.070+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f3c4c095ad0 con 0x7f3c5419bb00 2024-04-06T14:14:56.295 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:56 smithi090 ceph-mon[43420]: pgmap v295: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 11 MiB/s wr, 743 op/s 2024-04-06T14:14:56.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:56 smithi134 ceph-mon[47549]: pgmap v295: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 11 MiB/s wr, 743 op/s 2024-04-06T14:14:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.449+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c54198520 con 0x7f3c5419bb00 2024-04-06T14:14:56.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.449+0000 7f3c517fa700 1 -- 172.21.15.90:0/513533807 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f3c4c0990f0 con 0x7f3c5419bb00 2024-04-06T14:14:56.450 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:14:56.450 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.452+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 msgr2=0x7f3c3c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.452+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 0x7f3c3c0a3750 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f3c440098c0 tx=0x7f3c44019040 comp rx=0 tx=0).stop 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.452+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 msgr2=0x7f3c5419bf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.452+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 secure :-1 s=READY pgs=703 cs=0 l=1 rev1=1 crypto rx=0x7f3c4c00c110 tx=0x7f3c4c00c420 comp rx=0 tx=0).stop 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 shutdown_connections 2024-04-06T14:14:56.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c3c0a1290 0x7f3c3c0a3750 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c5419bb00 0x7f3c5419bf80 unknown :-1 s=CLOSED pgs=703 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c54100e90 0x7f3c54197270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 --2- 172.21.15.90:0/513533807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c54105fb0 0x7f3c541977b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 >> 172.21.15.90:0/513533807 conn(0x7f3c540fc750 msgr2=0x7f3c540fdbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 shutdown_connections 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.453+0000 7f3c5a858700 1 -- 172.21.15.90:0/513533807 wait complete. 2024-04-06T14:14:56.454 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:14:56.469 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:56.469 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:14:56.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:56 smithi105 ceph-mon[46969]: pgmap v295: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 11 MiB/s wr, 743 op/s 2024-04-06T14:14:56.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 -- 172.21.15.90:0/2290131996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4934105da0 msgr2=0x7f4934106180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 --2- 172.21.15.90:0/2290131996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4934105da0 0x7f4934106180 secure :-1 s=READY pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f49240099c0 tx=0x7f4924009cd0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 -- 172.21.15.90:0/2290131996 shutdown_connections 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 --2- 172.21.15.90:0/2290131996 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f493410cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 --2- 172.21.15.90:0/2290131996 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4934105da0 0x7f4934106180 secure :-1 s=CLOSED pgs=704 cs=0 l=1 rev1=1 crypto rx=0x7f49240099c0 tx=0x7f4924009cd0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 --2- 172.21.15.90:0/2290131996 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 -- 172.21.15.90:0/2290131996 >> 172.21.15.90:0/2290131996 conn(0x7f49340769b0 msgr2=0x7f4934076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.671+0000 7f4939d6c700 1 -- 172.21.15.90:0/2290131996 shutdown_connections 2024-04-06T14:14:56.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.672+0000 7f4939d6c700 1 -- 172.21.15.90:0/2290131996 wait complete. 2024-04-06T14:14:56.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.672+0000 7f4939d6c700 1 Processor -- start 2024-04-06T14:14:56.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 -- start start 2024-04-06T14:14:56.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934108e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 0x7f49341a9a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49341a9fe0 con 0x7f49341098d0 2024-04-06T14:14:56.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49341aa150 con 0x7f49340fff00 2024-04-06T14:14:56.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4939d6c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49341aa310 con 0x7f49341008c0 2024-04-06T14:14:56.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4933fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 0x7f49341a9a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57116/0 (socket says 172.21.15.90:57116) 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 -- 172.21.15.90:0/800269904 learned_addr learned my addr 172.21.15.90:0/800269904 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f49337fe700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934108e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 -- 172.21.15.90:0/800269904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 msgr2=0x7f4934108e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934108e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 -- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 msgr2=0x7f49341a9a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 0x7f49341a9a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4932ffd700 1 -- 172.21.15.90:0/800269904 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f49240095f0 con 0x7f49341008c0 2024-04-06T14:14:56.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f4933fff700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 0x7f49341a9a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.673+0000 7f49337fe700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4932ffd700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f492800e9a0 tx=0x7f492800ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4930ff9700 1 -- 172.21.15.90:0/800269904 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4928009790 con 0x7f49341008c0 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4930ff9700 1 -- 172.21.15.90:0/800269904 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4928010430 con 0x7f49341008c0 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f49341aa5f0 con 0x7f49341008c0 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4930ff9700 1 -- 172.21.15.90:0/800269904 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4928018680 con 0x7f49341008c0 2024-04-06T14:14:56.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.674+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49341aab00 con 0x7f49341008c0 2024-04-06T14:14:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.677+0000 7f4930ff9700 1 -- 172.21.15.90:0/800269904 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f49280105a0 con 0x7f49341008c0 2024-04-06T14:14:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.677+0000 7f4930ff9700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 0x7f491c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.677+0000 7f4930ff9700 1 -- 172.21.15.90:0/800269904 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4928014070 con 0x7f49341008c0 2024-04-06T14:14:56.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.677+0000 7f49337fe700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 0x7f491c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 Processor -- start 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 -- start start 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f49337fe700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 0x7f491c0a3750 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f49240058e0 tx=0x7f4924005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbccde6b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbccdf08a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccdf0de0 0x55cbccdf31d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccd3e990 con 0x55cbccde6780 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccd3c350 con 0x55cbccde70a0 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4945f20b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccd37c40 con 0x55cbccdf0de0 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f49337fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbccde6b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f49337fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbccde6b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47856/0 (socket says 172.21.15.90:47856) 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f49337fe700 1 -- 172.21.15.90:0/773821969 learned_addr learned my addr 172.21.15.90:0/773821969 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4933fff700 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccdf0de0 0x55cbccdf31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.678+0000 7f4932ffd700 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbccdf08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 -- 172.21.15.90:0/773821969 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccdf0de0 msgr2=0x55cbccdf31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccdf0de0 0x55cbccdf31d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 -- 172.21.15.90:0/773821969 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 msgr2=0x55cbccde6b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbccde6b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 -- 172.21.15.90:0/773821969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cbccbb93b0 con 0x55cbccde70a0 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4932ffd700 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbccdf08a0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f49280ca600 tx=0x7f4928099a10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=442e98b60d9e9325 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.679+0000 7f4919ffb700 1 -- 172.21.15.90:0/773821969 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f492809ddf0 con 0x55cbccde70a0 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.682+0000 7f4919ffb700 1 -- 172.21.15.90:0/773821969 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f49280ade90 con 0x55cbccde70a0 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.682+0000 7f4919ffb700 1 -- 172.21.15.90:0/773821969 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f49280994b0 con 0x55cbccde70a0 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.683+0000 7f4945f20b80 1 -- 172.21.15.90:0/773821969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 msgr2=0x55cbccdf08a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.683+0000 7f4945f20b80 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbccdf08a0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f49280ca600 tx=0x7f4928099a10 comp rx=0 tx=0).stop 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.685+0000 7f4945f20b80 1 -- 172.21.15.90:0/773821969 shutdown_connections 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.685+0000 7f4945f20b80 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccdf0de0 0x55cbccdf31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.685+0000 7f4945f20b80 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbccde6b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.685+0000 7f4945f20b80 1 --2- 172.21.15.90:0/773821969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbccdf08a0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.685+0000 7f4945f20b80 1 -- 172.21.15.90:0/773821969 >> 172.21.15.90:0/773821969 conn(0x55cbccd456a0 msgr2=0x55cbccde1a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.708 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.704+0000 7f4945f20b80 1 -- 172.21.15.90:0/773821969 shutdown_connections 2024-04-06T14:14:56.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.713+0000 7f4945f20b80 1 -- 172.21.15.90:0/773821969 wait complete. 2024-04-06T14:14:56.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.713+0000 7f4945f20b80 1 -- start start 2024-04-06T14:14:56.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.713+0000 7f4945f20b80 1 Processor -- start 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbccdedbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbcccd3d80 con 0x55cbccde70a0 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbcccd3a20 con 0x55cbccdf0de0 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccbbb6a0 con 0x55cbccde6780 2024-04-06T14:14:56.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.726+0000 7f4945f20b80 10 client.0 _ll_register_callbacks cb 0x55cbcccdd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:56.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.731+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.731+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47858/0 (socket says 172.21.15.90:47858) 2024-04-06T14:14:56.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.731+0000 7f4932ffd700 1 -- 172.21.15.90:0/1924535065 learned_addr learned my addr 172.21.15.90:0/1924535065 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.732+0000 7f4933fff700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.732+0000 7f49337fe700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbccdedbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.737+0000 7f4932ffd700 1 -- 172.21.15.90:0/1924535065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 msgr2=0x55cbccdedbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.737+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbccdedbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.737+0000 7f4932ffd700 1 -- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 msgr2=0x55cbccdee650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.737+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.737+0000 7f4932ffd700 1 -- 172.21.15.90:0/1924535065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49341aab00 con 0x55cbccde70a0 2024-04-06T14:14:56.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.740+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f49280964c0 tx=0x7f4928010ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f5b1d7da06e0c12c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.740+0000 7f4912ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:56.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.740+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f492809ddf0 con 0x55cbccde70a0 2024-04-06T14:14:56.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.740+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f492809a430 con 0x55cbccde70a0 2024-04-06T14:14:56.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.745+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cbcccf8380 con 0x55cbccde70a0 2024-04-06T14:14:56.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.750+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f49280b82a0 con 0x55cbccde70a0 2024-04-06T14:14:56.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.750+0000 7f4945f20b80 10 client.17001 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:56.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.750+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cbccb65b60 con 0x55cbccde70a0 2024-04-06T14:14:56.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4928063070 con 0x55cbccde70a0 2024-04-06T14:14:56.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4928098f20 con 0x55cbccde70a0 2024-04-06T14:14:56.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 10 client.17001 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:56.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 10 client.17001 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:14:56.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 20 client.17001 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:56.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce11760 0x55cbcce13b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 4 client.17001 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:56.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cbccd30700 con 0x55cbcce11760 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.753+0000 7f49337fe700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce11760 0x55cbcce13b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.754+0000 7f4912ffd700 10 client.17001 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.754+0000 7f49337fe700 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce11760 0x55cbcce13b50 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=27aaa77a4b088340 server_cookie=1512a8d8a3cbef59 in_seq=0 out_seq=0 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.754+0000 7f4912ffd700 1 -- 172.21.15.90:0/1924535065 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cbccd30700 con 0x55cbcce11760 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.754+0000 7f4912ffd700 10 client.17001 handle_command_reply: tid=0 2024-04-06T14:14:56.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 client.17001 shutdown 2024-04-06T14:14:56.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 msgr2=0x55cbccdee110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 secure :-1 s=READY pgs=705 cs=0 l=1 rev1=1 crypto rx=0x7f49280964c0 tx=0x7f4928010ba0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 shutdown_connections 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce11760 0x55cbcce13b50 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbccdedbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4912ffd700 0 client.17001 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1924535065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde70a0 0x55cbccdee110 unknown :-1 s=CLOSED pgs=705 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.755+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 >> 172.21.15.90:0/1924535065 conn(0x55cbccd456a0 msgr2=0x55cbccde0c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.762 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.762+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 shutdown_connections 2024-04-06T14:14:56.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.763+0000 7f4945f20b80 1 -- 172.21.15.90:0/1924535065 wait complete. 2024-04-06T14:14:56.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.766+0000 7f4945f20b80 20 client.17001 trim_cache size 0 max 16384 2024-04-06T14:14:56.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 Processor -- start 2024-04-06T14:14:56.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 -- start start 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde70a0 0x55cbccdee140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccdef240 con 0x55cbccde6780 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccdf0740 con 0x55cbccdf0de0 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4945f20b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbccde7a10 con 0x55cbccde70a0 2024-04-06T14:14:56.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f49337fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f49337fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47862/0 (socket says 172.21.15.90:47862) 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f49337fe700 1 -- 172.21.15.90:0/615231955 learned_addr learned my addr 172.21.15.90:0/615231955 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.782+0000 7f4933fff700 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 -- 172.21.15.90:0/615231955 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde70a0 msgr2=0x55cbccdee140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde70a0 0x55cbccdee140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 -- 172.21.15.90:0/615231955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 msgr2=0x55cbccdee680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 -- 172.21.15.90:0/615231955 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cbcccfdea0 con 0x55cbccde6780 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f49337fe700 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f4924000c00 tx=0x7f492401f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=192a21f8559056cc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f491a7fc700 1 -- 172.21.15.90:0/615231955 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f492402a040 con 0x55cbccde6780 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f491a7fc700 1 -- 172.21.15.90:0/615231955 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f492402b040 con 0x55cbccde6780 2024-04-06T14:14:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.783+0000 7f491a7fc700 1 -- 172.21.15.90:0/615231955 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4924003c90 con 0x55cbccde6780 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 -- 172.21.15.90:0/615231955 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 msgr2=0x55cbcce0dbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 secure :-1 s=READY pgs=706 cs=0 l=1 rev1=1 crypto rx=0x7f4924000c00 tx=0x7f492401f000 comp rx=0 tx=0).stop 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 -- 172.21.15.90:0/615231955 shutdown_connections 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccdf0de0 0x55cbccdee680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccde6780 0x55cbcce0dbc0 unknown :-1 s=CLOSED pgs=706 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 --2- 172.21.15.90:0/615231955 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde70a0 0x55cbccdee140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 -- 172.21.15.90:0/615231955 >> 172.21.15.90:0/615231955 conn(0x55cbccd456a0 msgr2=0x55cbccdec4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 -- 172.21.15.90:0/615231955 shutdown_connections 2024-04-06T14:14:56.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.784+0000 7f4945f20b80 1 -- 172.21.15.90:0/615231955 wait complete. 2024-04-06T14:14:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 -- start start 2024-04-06T14:14:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 Processor -- start 2024-04-06T14:14:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbcce089c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 0x55cbcce073c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbcce079a0 con 0x55cbccdf0de0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbcce07b10 con 0x55cbccde70a0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cbcce07cd0 con 0x55cbccde6780 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4945f20b80 10 client.0 _ll_register_callbacks cb 0x55cbcccdd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f49337fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbcce089c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4932ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34564/0 (socket says 172.21.15.90:34564) 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.785+0000 7f4932ffd700 1 -- 172.21.15.90:0/1775646558 learned_addr learned my addr 172.21.15.90:0/1775646558 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4933fff700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 0x55cbcce073c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 -- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 msgr2=0x55cbcce06e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 -- 172.21.15.90:0/1775646558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 msgr2=0x55cbcce073c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 0x55cbcce073c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 -- 172.21.15.90:0/1775646558 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f493410d360 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4933fff700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 0x55cbcce073c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f49337fe700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbcce089c0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f49240095c0 tx=0x7f492400be70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8dd1cc345a2641b1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4912ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f492402a040 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f49280ade90 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4924004690 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f49240332b0 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4945f20b80 10 client.26485 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.786+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cbccbbdcd0 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f49280ade90 con 0x55cbccde6780 2024-04-06T14:14:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f492403e730 con 0x55cbccde6780 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 10 client.26485 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 10 client.26485 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 20 client.26485 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce77790 0x55cbcce79bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 4 client.26485 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.787+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55cbccb86cf0 con 0x55cbcce77790 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.788+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce77790 0x55cbcce79bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.788+0000 7f4912ffd700 10 client.26485 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.788+0000 7f4932ffd700 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce77790 0x55cbcce79bb0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=5bce091b78391937 server_cookie=94e27b5aee9a629 in_seq=0 out_seq=0 2024-04-06T14:14:56.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.789+0000 7f4912ffd700 1 -- 172.21.15.90:0/1775646558 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55cbccb86cf0 con 0x55cbcce77790 2024-04-06T14:14:56.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.789+0000 7f4912ffd700 10 client.26485 handle_command_reply: tid=0 2024-04-06T14:14:56.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.789+0000 7f4945f20b80 1 client.26485 shutdown 2024-04-06T14:14:56.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 msgr2=0x55cbcce089c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbcce089c0 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f49240095c0 tx=0x7f492400be70 comp rx=0 tx=0).stop 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 shutdown_connections 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55cbcce77790 0x55cbcce79bb0 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cbccdf0de0 0x55cbcce073c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cbccde6780 0x55cbcce089c0 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 --2- 172.21.15.90:0/1775646558 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cbccde70a0 0x55cbcce06e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.791+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 >> 172.21.15.90:0/1775646558 conn(0x55cbccd456a0 msgr2=0x55cbccde5770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.793+0000 7f4912ffd700 0 client.26485 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:14:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.793+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 shutdown_connections 2024-04-06T14:14:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.793+0000 7f4945f20b80 1 -- 172.21.15.90:0/1775646558 wait complete. 2024-04-06T14:14:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.794+0000 7f4945f20b80 20 client.26485 trim_cache size 0 max 16384 2024-04-06T14:14:56.795 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.794+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 msgr2=0x7f491c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.794+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 0x7f491c0a3750 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f49240058e0 tx=0x7f4924005fb0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.794+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 msgr2=0x7f4934109390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.794+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f492800e9a0 tx=0x7f492800ecb0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 shutdown_connections 2024-04-06T14:14:56.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f491c0a1290 0x7f491c0a3750 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f49341098d0 0x7f49341a9a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49340fff00 0x7f4934108e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 --2- 172.21.15.90:0/800269904 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f49341008c0 0x7f4934109390 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 >> 172.21.15.90:0/800269904 conn(0x7f49340769b0 msgr2=0x7f49340fef10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 shutdown_connections 2024-04-06T14:14:56.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.795+0000 7f4939d6c700 1 -- 172.21.15.90:0/800269904 wait complete. 2024-04-06T14:14:56.811 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:56.811 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:14:56.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 -- 172.21.15.90:0/1968525585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 msgr2=0x7fa714100980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 --2- 172.21.15.90:0/1968525585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa714100980 secure :-1 s=READY pgs=707 cs=0 l=1 rev1=1 crypto rx=0x7fa710009990 tx=0x7fa710009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 -- 172.21.15.90:0/1968525585 shutdown_connections 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 --2- 172.21.15.90:0/1968525585 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 0x7fa71410cd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 --2- 172.21.15.90:0/1968525585 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa714100980 unknown :-1 s=CLOSED pgs=707 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 --2- 172.21.15.90:0/1968525585 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714100f50 0x7fa714104a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.984+0000 7fa71bae0700 1 -- 172.21.15.90:0/1968525585 >> 172.21.15.90:0/1968525585 conn(0x7fa7140761b0 msgr2=0x7fa7140fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- 172.21.15.90:0/1968525585 shutdown_connections 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- 172.21.15.90:0/1968525585 wait complete. 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 Processor -- start 2024-04-06T14:14:56.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- start start 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 0x7fa7141956f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714199a90 0x7fa714199f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa71419a510 con 0x7fa7141005a0 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa71419a680 con 0x7fa7141051a0 2024-04-06T14:14:56.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.985+0000 7fa71bae0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa71419a840 con 0x7fa714199a90 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47882/0 (socket says 172.21.15.90:47882) 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 -- 172.21.15.90:0/3465725108 learned_addr learned my addr 172.21.15.90:0/3465725108 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71a07d700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714199a90 0x7fa714199f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71907b700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 0x7fa7141956f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 -- 172.21.15.90:0/3465725108 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714199a90 msgr2=0x7fa714199f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714199a90 0x7fa714199f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 -- 172.21.15.90:0/3465725108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 msgr2=0x7fa7141956f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 0x7fa7141956f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 -- 172.21.15.90:0/3465725108 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7100095f0 con 0x7fa7141005a0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.986+0000 7fa71987c700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa710005910 tx=0x7fa710004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.987+0000 7fa70affd700 1 -- 172.21.15.90:0/3465725108 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa71001d070 con 0x7fa7141005a0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.987+0000 7fa70affd700 1 -- 172.21.15.90:0/3465725108 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa710022470 con 0x7fa7141005a0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.987+0000 7fa70affd700 1 -- 172.21.15.90:0/3465725108 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa710017650 con 0x7fa7141005a0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.988+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7141a1d40 con 0x7fa7141005a0 2024-04-06T14:14:56.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.988+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7141a2110 con 0x7fa7141005a0 2024-04-06T14:14:56.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.991+0000 7fa70affd700 1 -- 172.21.15.90:0/3465725108 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa7100225e0 con 0x7fa7141005a0 2024-04-06T14:14:56.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.992+0000 7fa70affd700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 0x7fa7000a7bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.992+0000 7fa70affd700 1 -- 172.21.15.90:0/3465725108 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa7100cd9b0 con 0x7fa7141005a0 2024-04-06T14:14:56.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.992+0000 7fa71907b700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 0x7fa7000a7bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.993+0000 7fa71907b700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 0x7fa7000a7bf0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fa714196700 tx=0x7fa704009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.993+0000 7fa727c94b80 1 Processor -- start 2024-04-06T14:14:56.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.993+0000 7fa727c94b80 1 -- start start 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.993+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 0x563c8e6908a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e6931d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa727c94b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e5de990 con 0x563c8e690de0 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa727c94b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e5dc350 con 0x563c8e6870a0 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa727c94b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e5d7c40 con 0x563c8e686780 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57176/0 (socket says 172.21.15.90:57176) 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 -- 172.21.15.90:0/3013985933 learned_addr learned my addr 172.21.15.90:0/3013985933 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71907b700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 0x563c8e6908a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71a07d700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e6931d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 -- 172.21.15.90:0/3013985933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 msgr2=0x563c8e6908a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 0x563c8e6908a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 -- 172.21.15.90:0/3013985933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 msgr2=0x563c8e6931d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:56.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e6931d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71987c700 1 -- 172.21.15.90:0/3013985933 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563c8e4593b0 con 0x563c8e686780 2024-04-06T14:14:56.996 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.994+0000 7fa71907b700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 0x563c8e6908a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.996+0000 7fa71987c700 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fa7100975a0 tx=0x7fa710009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=426928a78234c1a3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.996+0000 7fa6fffff700 1 -- 172.21.15.90:0/3013985933 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa71009b230 con 0x563c8e686780 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.996+0000 7fa6fffff700 1 -- 172.21.15.90:0/3013985933 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa71009ab30 con 0x563c8e686780 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.996+0000 7fa6fffff700 1 -- 172.21.15.90:0/3013985933 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa71009ad90 con 0x563c8e686780 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- 172.21.15.90:0/3013985933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 msgr2=0x563c8e686b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fa7100975a0 tx=0x7fa710009310 comp rx=0 tx=0).stop 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- 172.21.15.90:0/3013985933 shutdown_connections 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e6931d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e686b60 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.998 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3013985933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e6870a0 0x563c8e6908a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:56.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- 172.21.15.90:0/3013985933 >> 172.21.15.90:0/3013985933 conn(0x563c8e5e56a0 msgr2=0x563c8e681a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:56.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- 172.21.15.90:0/3013985933 shutdown_connections 2024-04-06T14:14:56.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- 172.21.15.90:0/3013985933 wait complete. 2024-04-06T14:14:56.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 -- start start 2024-04-06T14:14:56.999 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.997+0000 7fa727c94b80 1 Processor -- start 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 0x563c8e6911c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e691700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e69c240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e573d80 con 0x563c8e686780 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e573a20 con 0x563c8e690de0 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e45b6a0 con 0x563c8e6870a0 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa727c94b80 10 client.0 _ll_register_callbacks cb 0x563c8e57d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71987c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 0x563c8e6911c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e691700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71987c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 0x563c8e6911c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47900/0 (socket says 172.21.15.90:47900) 2024-04-06T14:14:57.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71987c700 1 -- 172.21.15.90:0/846492297 learned_addr learned my addr 172.21.15.90:0/846492297 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71a07d700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e69c240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 -- 172.21.15.90:0/846492297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 msgr2=0x563c8e69c240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e69c240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 -- 172.21.15.90:0/846492297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 msgr2=0x563c8e6911c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 0x563c8e6911c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.998+0000 7fa71907b700 1 -- 172.21.15.90:0/846492297 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa704017070 con 0x563c8e6870a0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa71907b700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e691700 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fa7140734a0 tx=0x7fa7040047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a9ce0211009e0748 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa704022070 con 0x563c8e6870a0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa704004240 con 0x563c8e6870a0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563c8e598380 con 0x563c8e6870a0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa704004400 con 0x563c8e6870a0 2024-04-06T14:14:57.002 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 10 client.26497 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fa7141a2110 con 0x563c8e6870a0 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563c8e598380 con 0x563c8e6870a0 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa704030750 con 0x563c8e6870a0 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 10 client.26497 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 10 client.26497 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 20 client.26497 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e6a05a0 0x563c8e6a09b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 4 client.26497 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:56.999+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563c8e5d0700 con 0x563c8e6a05a0 2024-04-06T14:14:57.003 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.000+0000 7fa71987c700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e6a05a0 0x563c8e6a09b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.001+0000 7fa71987c700 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e6a05a0 0x563c8e6a09b0 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=94794a4c7c170374 server_cookie=a6ef5d9cbf528 in_seq=0 out_seq=0 2024-04-06T14:14:57.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.001+0000 7fa6fd7fa700 10 client.26497 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.002+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/846492297 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563c8e5d0700 con 0x563c8e6a05a0 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.002+0000 7fa6fd7fa700 10 client.26497 handle_command_reply: tid=0 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.002+0000 7fa727c94b80 1 client.26497 shutdown 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 msgr2=0x563c8e691700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e691700 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fa7140734a0 tx=0x7fa7040047c0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 shutdown_connections 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e6a05a0 0x563c8e6a09b0 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e69c240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e686780 0x563c8e6911c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 --2- 172.21.15.90:0/846492297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e691700 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 >> 172.21.15.90:0/846492297 conn(0x563c8e5e56a0 msgr2=0x563c8e680c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa6fd7fa700 0 client.26497 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 shutdown_connections 2024-04-06T14:14:57.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.003+0000 7fa727c94b80 1 -- 172.21.15.90:0/846492297 wait complete. 2024-04-06T14:14:57.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.005+0000 7fa727c94b80 20 client.26497 trim_cache size 0 max 16384 2024-04-06T14:14:57.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.022+0000 7fa727c94b80 1 Processor -- start 2024-04-06T14:14:57.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 -- start start 2024-04-06T14:14:57.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e686780 0x563c8e6ca220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e68edd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e6ca760 con 0x563c8e690de0 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e691ee0 con 0x563c8e686780 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa727c94b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e68e510 con 0x563c8e6870a0 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57192/0 (socket says 172.21.15.90:57192) 2024-04-06T14:14:57.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71a07d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e68edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 -- 172.21.15.90:0/315238954 learned_addr learned my addr 172.21.15.90:0/315238954 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71987c700 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e686780 0x563c8e6ca220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 -- 172.21.15.90:0/315238954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e686780 msgr2=0x563c8e6ca220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e686780 0x563c8e6ca220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 -- 172.21.15.90:0/315238954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 msgr2=0x563c8e68edd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e68edd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.023+0000 7fa71907b700 1 -- 172.21.15.90:0/315238954 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563c8e405b60 con 0x563c8e6870a0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa71907b700 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fa704011b40 tx=0x7fa70401dd20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5463aabe653033ba server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa708da5700 1 -- 172.21.15.90:0/315238954 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa704022040 con 0x563c8e6870a0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa708da5700 1 -- 172.21.15.90:0/315238954 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa704030450 con 0x563c8e6870a0 2024-04-06T14:14:57.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa708da5700 1 -- 172.21.15.90:0/315238954 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa704003c40 con 0x563c8e6870a0 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa727c94b80 1 -- 172.21.15.90:0/315238954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 msgr2=0x563c8e68e890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.024+0000 7fa727c94b80 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fa704011b40 tx=0x7fa70401dd20 comp rx=0 tx=0).stop 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.025+0000 7fa727c94b80 1 -- 172.21.15.90:0/315238954 shutdown_connections 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.025+0000 7fa727c94b80 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e690de0 0x563c8e68edd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.025+0000 7fa727c94b80 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e686780 0x563c8e6ca220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.025+0000 7fa727c94b80 1 --2- 172.21.15.90:0/315238954 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e6870a0 0x563c8e68e890 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.025+0000 7fa727c94b80 1 -- 172.21.15.90:0/315238954 >> 172.21.15.90:0/315238954 conn(0x563c8e5e56a0 msgr2=0x563c8e68b2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.026+0000 7fa727c94b80 1 -- 172.21.15.90:0/315238954 shutdown_connections 2024-04-06T14:14:57.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.026+0000 7fa727c94b80 1 -- 172.21.15.90:0/315238954 wait complete. 2024-04-06T14:14:57.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.026+0000 7fa727c94b80 1 -- start start 2024-04-06T14:14:57.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.026+0000 7fa727c94b80 1 Processor -- start 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.026+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e6c5700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 0x563c8e6c5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e6c6180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e6ca5d0 con 0x563c8e6870a0 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e7162b0 con 0x563c8e690de0 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563c8e716420 con 0x563c8e686780 2024-04-06T14:14:57.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa727c94b80 10 client.0 _ll_register_callbacks cb 0x563c8e57d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71987c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e6c5700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e6c6180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71907b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 0x563c8e6c5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71987c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e6c5700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57208/0 (socket says 172.21.15.90:57208) 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71987c700 1 -- 172.21.15.90:0/3447027708 learned_addr learned my addr 172.21.15.90:0/3447027708 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 -- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 msgr2=0x563c8e6c5700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e6c5700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 -- 172.21.15.90:0/3447027708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 msgr2=0x563c8e6c5c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 0x563c8e6c5c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 -- 172.21.15.90:0/3447027708 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7141a2110 con 0x563c8e690de0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71907b700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 0x563c8e6c5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa71a07d700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e6c6180 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x563c8e6881f0 tx=0x7fa70c026410 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a70670df93a1b79f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.027+0000 7fa6fd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa70c02c070 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa70c026c50 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa71009ab30 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa70c026e10 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 10 client.26720 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563c8e59dea0 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa71009ab30 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa70c0058f0 con 0x563c8e690de0 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 10 client.26720 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 10 client.26720 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 20 client.26720 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e717c00 0x563c8e71a020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 4 client.26720 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:57.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.028+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x563c8e426cf0 con 0x563c8e717c00 2024-04-06T14:14:57.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.031+0000 7fa71987c700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e717c00 0x563c8e71a020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa6fd7fa700 10 client.26720 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:57.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa71987c700 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e717c00 0x563c8e71a020 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e3ab9d84a13a1178 server_cookie=1d4738b6689c27f0 in_seq=0 out_seq=0 2024-04-06T14:14:57.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa6fd7fa700 1 -- 172.21.15.90:0/3447027708 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x563c8e426cf0 con 0x563c8e717c00 2024-04-06T14:14:57.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa6fd7fa700 10 client.26720 handle_command_reply: tid=0 2024-04-06T14:14:57.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 client.26720 shutdown 2024-04-06T14:14:57.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 msgr2=0x563c8e6c6180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e6c6180 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x563c8e6881f0 tx=0x7fa70c026410 comp rx=0 tx=0).stop 2024-04-06T14:14:57.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 shutdown_connections 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x563c8e717c00 0x563c8e71a020 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563c8e690de0 0x563c8e6c6180 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563c8e686780 0x563c8e6c5700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa6fd7fa700 0 client.26720 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 --2- 172.21.15.90:0/3447027708 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563c8e6870a0 0x563c8e6c5c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.032+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 >> 172.21.15.90:0/3447027708 conn(0x563c8e5e56a0 msgr2=0x563c8e68e7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.033+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 shutdown_connections 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.033+0000 7fa727c94b80 1 -- 172.21.15.90:0/3447027708 wait complete. 2024-04-06T14:14:57.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.033+0000 7fa727c94b80 20 client.26720 trim_cache size 0 max 16384 2024-04-06T14:14:57.035 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:57.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 msgr2=0x7fa7000a7bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 0x7fa7000a7bf0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fa714196700 tx=0x7fa704009260 comp rx=0 tx=0).stop 2024-04-06T14:14:57.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 msgr2=0x7fa7141951b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 secure :-1 s=READY pgs=708 cs=0 l=1 rev1=1 crypto rx=0x7fa710005910 tx=0x7fa710004730 comp rx=0 tx=0).stop 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 shutdown_connections 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa7000a5730 0x7fa7000a7bf0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa714199a90 0x7fa714199f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa7141005a0 0x7fa7141951b0 unknown :-1 s=CLOSED pgs=708 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 --2- 172.21.15.90:0/3465725108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa7141051a0 0x7fa7141956f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 >> 172.21.15.90:0/3465725108 conn(0x7fa7140761b0 msgr2=0x7fa7140fb860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 shutdown_connections 2024-04-06T14:14:57.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.034+0000 7fa71bae0700 1 -- 172.21.15.90:0/3465725108 wait complete. 2024-04-06T14:14:57.051 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:14:57.051 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:14:57.079 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:57 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/513533807' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/3298514884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68103b90 msgr2=0x7fbd68107bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/3298514884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68103b90 0x7fbd68107bd0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c009990 tx=0x7fbd5c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/3298514884 shutdown_connections 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/3298514884 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd68108300 0x7fbd6810fe80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/3298514884 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd681031e0 0x7fbd681035c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/3298514884 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68103b90 0x7fbd68107bd0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/3298514884 >> 172.21.15.90:0/3298514884 conn(0x7fbd680fea40 msgr2=0x7fbd68100e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/3298514884 shutdown_connections 2024-04-06T14:14:57.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.235+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/3298514884 wait complete. 2024-04-06T14:14:57.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 Processor -- start 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 -- start start 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd681031e0 0x7fbd68199480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd6819dd10 0x7fbd6819e190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd6819e790 con 0x7fbd6819dd10 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd6819e900 con 0x7fbd68108300 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd6d3dc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd6819eac0 con 0x7fbd681031e0 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd66ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd681031e0 0x7fbd68199480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34624/0 (socket says 172.21.15.90:34624) 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd667fc700 1 -- 172.21.15.90:0/2959821121 learned_addr learned my addr 172.21.15.90:0/2959821121 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.236+0000 7fbd677fe700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd6819dd10 0x7fbd6819e190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 -- 172.21.15.90:0/2959821121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd681031e0 msgr2=0x7fbd68199480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd681031e0 0x7fbd68199480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 -- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd6819dd10 msgr2=0x7fbd6819e190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd6819dd10 0x7fbd6819e190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 -- 172.21.15.90:0/2959821121 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd5c0095f0 con 0x7fbd68108300 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c003730 tx=0x7fbd5c00f690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd4ffff700 1 -- 172.21.15.90:0/2959821121 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c01d070 con 0x7fbd68108300 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd4ffff700 1 -- 172.21.15.90:0/2959821121 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd5c0054f0 con 0x7fbd68108300 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd4ffff700 1 -- 172.21.15.90:0/2959821121 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c004b30 con 0x7fbd68108300 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd681a5fc0 con 0x7fbd68108300 2024-04-06T14:14:57.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.237+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd681a6390 con 0x7fbd68108300 2024-04-06T14:14:57.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.241+0000 7fbd4ffff700 1 -- 172.21.15.90:0/2959821121 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fbd5c004c90 con 0x7fbd68108300 2024-04-06T14:14:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.241+0000 7fbd4ffff700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 0x7fbd500a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.241+0000 7fbd4ffff700 1 -- 172.21.15.90:0/2959821121 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbd5c0cda10 con 0x7fbd68108300 2024-04-06T14:14:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 0x7fbd500a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 Processor -- start 2024-04-06T14:14:57.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 -- start start 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41519b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 0x55fb415238a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb415261d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41471990 con 0x55fb4151a0a0 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb4146f350 con 0x55fb41519780 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd79590b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb4146ac40 con 0x55fb41523de0 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 0x7fbd500a37d0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fbd58005950 tx=0x7fbd5800b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd66ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41519b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 0x55fb415238a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd66ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41519b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34632/0 (socket says 172.21.15.90:34632) 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd66ffd700 1 -- 172.21.15.90:0/1695831469 learned_addr learned my addr 172.21.15.90:0/1695831469 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 -- 172.21.15.90:0/1695831469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 msgr2=0x55fb415261d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb415261d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 -- 172.21.15.90:0/1695831469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 msgr2=0x55fb41519b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41519b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.242+0000 7fbd667fc700 1 -- 172.21.15.90:0/1695831469 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fb412ec3b0 con 0x55fb4151a0a0 2024-04-06T14:14:57.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.243+0000 7fbd667fc700 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 0x55fb415238a0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c0cb140 tx=0x7fbd5c09a830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84f07fd1feefbc9c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.243+0000 7fbd4d7fa700 1 -- 172.21.15.90:0/1695831469 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c09e9c0 con 0x55fb4151a0a0 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.243+0000 7fbd4d7fa700 1 -- 172.21.15.90:0/1695831469 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd5c0a2f80 con 0x55fb4151a0a0 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.243+0000 7fbd4d7fa700 1 -- 172.21.15.90:0/1695831469 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c09bba0 con 0x55fb4151a0a0 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- 172.21.15.90:0/1695831469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 msgr2=0x55fb415238a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 0x55fb415238a0 secure :-1 s=READY pgs=709 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c0cb140 tx=0x7fbd5c09a830 comp rx=0 tx=0).stop 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- 172.21.15.90:0/1695831469 shutdown_connections 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb415261d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41519b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 --2- 172.21.15.90:0/1695831469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb4151a0a0 0x55fb415238a0 unknown :-1 s=CLOSED pgs=709 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- 172.21.15.90:0/1695831469 >> 172.21.15.90:0/1695831469 conn(0x55fb414786a0 msgr2=0x55fb41514a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- 172.21.15.90:0/1695831469 shutdown_connections 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- 172.21.15.90:0/1695831469 wait complete. 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 -- start start 2024-04-06T14:14:57.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.244+0000 7fbd79590b80 1 Processor -- start 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb415237c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb41520ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb41521110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41406d80 con 0x55fb41523de0 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41406a20 con 0x55fb41519780 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb412ee6a0 con 0x55fb4151a0a0 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd79590b80 10 client.0 _ll_register_callbacks cb 0x55fb41410470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd677fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb41521110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd677fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb41521110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47942/0 (socket says 172.21.15.90:47942) 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd677fe700 1 -- 172.21.15.90:0/3281523198 learned_addr learned my addr 172.21.15.90:0/3281523198 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb41520ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.247 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.245+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb415237c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 -- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 msgr2=0x55fb415237c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb415237c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 -- 172.21.15.90:0/3281523198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 msgr2=0x55fb41521110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb41521110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 -- 172.21.15.90:0/3281523198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd681a6390 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb41520ab0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c0cb170 tx=0x7fbd5c00fde0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9edc683d351008c6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd4aff5700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c0b34b0 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbd5c09e9c0 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55fb4142b380 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbd5c09bba0 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd79590b80 10 client.26512 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55fb41298b60 con 0x55fb4151a0a0 2024-04-06T14:14:57.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.246+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55fb4142b380 con 0x55fb4151a0a0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbd5c004fd0 con 0x55fb4151a0a0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 10 client.26512 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 10 client.26512 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 20 client.26512 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb415441f0 0x55fb415465e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 4 client.26512 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55fb41463700 con 0x55fb415441f0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb415441f0 0x55fb415465e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb415441f0 0x55fb415465e0 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9358183a2bdf1172 server_cookie=32e76cf2a3a64d1c in_seq=0 out_seq=0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.247+0000 7fbd4aff5700 10 client.26512 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd4aff5700 1 -- 172.21.15.90:0/3281523198 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55fb41463700 con 0x55fb415441f0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd4aff5700 10 client.26512 handle_command_reply: tid=0 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 client.26512 shutdown 2024-04-06T14:14:57.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 msgr2=0x55fb41520ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb41520ab0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c0cb170 tx=0x7fbd5c00fde0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 shutdown_connections 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb41521110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb415237c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb415441f0 0x55fb415465e0 crc :-1 s=READY pgs=171 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3281523198 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb41520ab0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd4aff5700 0 client.26512 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 >> 172.21.15.90:0/3281523198 conn(0x55fb414786a0 msgr2=0x55fb41513c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 shutdown_connections 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 1 -- 172.21.15.90:0/3281523198 wait complete. 2024-04-06T14:14:57.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.248+0000 7fbd79590b80 20 client.26512 trim_cache size 0 max 16384 2024-04-06T14:14:57.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.266+0000 7fbd79590b80 1 Processor -- start 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.266+0000 7fbd79590b80 1 -- start start 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.266+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41540810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb4151fff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd79590b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41515e00 con 0x55fb41523de0 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd79590b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb4151aa40 con 0x55fb41519780 2024-04-06T14:14:57.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd79590b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb4152ad90 con 0x55fb4151a0a0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd677fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb4151fff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57228/0 (socket says 172.21.15.90:57228) 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 -- 172.21.15.90:0/3092953881 learned_addr learned my addr 172.21.15.90:0/3092953881 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41540810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 -- 172.21.15.90:0/3092953881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 msgr2=0x55fb41540810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41540810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 -- 172.21.15.90:0/3092953881 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 msgr2=0x55fb4151fff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb4151fff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 -- 172.21.15.90:0/3092953881 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fb41430ea0 con 0x55fb4151a0a0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.267+0000 7fbd667fc700 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c005f50 tx=0x7fbd5c09a830 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3d4785714ff66977 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd4dffb700 1 -- 172.21.15.90:0/3092953881 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c09a410 con 0x55fb4151a0a0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd4dffb700 1 -- 172.21.15.90:0/3092953881 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd5c0b34b0 con 0x55fb4151a0a0 2024-04-06T14:14:57.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd4dffb700 1 -- 172.21.15.90:0/3092953881 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c099bd0 con 0x55fb4151a0a0 2024-04-06T14:14:57.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 -- 172.21.15.90:0/3092953881 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 msgr2=0x55fb4152a7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c005f50 tx=0x7fbd5c09a830 comp rx=0 tx=0).stop 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 -- 172.21.15.90:0/3092953881 shutdown_connections 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41523de0 0x55fb4151fff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb41519780 0x55fb41540810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 --2- 172.21.15.90:0/3092953881 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb4151a0a0 0x55fb4152a7d0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.268+0000 7fbd79590b80 1 -- 172.21.15.90:0/3092953881 >> 172.21.15.90:0/3092953881 conn(0x55fb414786a0 msgr2=0x55fb41510890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- 172.21.15.90:0/3092953881 shutdown_connections 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- 172.21.15.90:0/3092953881 wait complete. 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- start start 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 Processor -- start 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 0x55fb415289a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb41596f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb415295c0 con 0x55fb41519780 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41597470 con 0x55fb4151a0a0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fb41597630 con 0x55fb41523de0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.269+0000 7fbd79590b80 10 client.0 _ll_register_callbacks cb 0x55fb41410470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34660/0 (socket says 172.21.15.90:34660) 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 -- 172.21.15.90:0/2591285118 learned_addr learned my addr 172.21.15.90:0/2591285118 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd677fe700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb41596f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 0x55fb415289a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 -- 172.21.15.90:0/2591285118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 msgr2=0x55fb41596f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb41596f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 -- 172.21.15.90:0/2591285118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 msgr2=0x55fb415289a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 0x55fb415289a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 -- 172.21.15.90:0/2591285118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55fb41298b60 con 0x55fb4151a0a0 2024-04-06T14:14:57.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 0x55fb415289a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:57.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd667fc700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c098fc0 tx=0x7fbd5c009ee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b9dafb96c827f2a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:57.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd4aff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:57.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd5c09bda0 con 0x55fb4151a0a0 2024-04-06T14:14:57.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbd5c096a60 con 0x55fb4151a0a0 2024-04-06T14:14:57.274 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.270+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbd5c0a2f80 con 0x55fb4151a0a0 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.272+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbd5c096c20 con 0x55fb4151a0a0 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.272+0000 7fbd79590b80 10 client.26738 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.272+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55fb412f0cd0 con 0x55fb4151a0a0 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd5c0a2f80 con 0x55fb4151a0a0 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbd5c0ba160 con 0x55fb4151a0a0 2024-04-06T14:14:57.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 10 client.26738 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 10 client.26738 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 20 client.26738 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb41598890 0x55fb4159acb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 4 client.26738 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.273+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55fb412b9cf0 con 0x55fb41598890 2024-04-06T14:14:57.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb41598890 0x55fb4159acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:57.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd66ffd700 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb41598890 0x55fb4159acb0 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4d2b78508d234e86 server_cookie=fbe611b20b9307f9 in_seq=0 out_seq=0 2024-04-06T14:14:57.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd4aff5700 10 client.26738 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd4aff5700 1 -- 172.21.15.90:0/2591285118 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55fb412b9cf0 con 0x55fb41598890 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd4aff5700 10 client.26738 handle_command_reply: tid=0 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.277+0000 7fbd79590b80 1 client.26738 shutdown 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 msgr2=0x55fb41528ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c098fc0 tx=0x7fbd5c009ee0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 shutdown_connections 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55fb41598890 0x55fb4159acb0 crc :-1 s=READY pgs=172 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55fb41523de0 0x55fb41596f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd4aff5700 0 client.26738 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55fb41519780 0x55fb415289a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 --2- 172.21.15.90:0/2591285118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55fb4151a0a0 0x55fb41528ee0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.278+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 >> 172.21.15.90:0/2591285118 conn(0x55fb414786a0 msgr2=0x55fb41514440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.279+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 shutdown_connections 2024-04-06T14:14:57.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.279+0000 7fbd79590b80 1 -- 172.21.15.90:0/2591285118 wait complete. 2024-04-06T14:14:57.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.279+0000 7fbd79590b80 20 client.26738 trim_cache size 0 max 16384 2024-04-06T14:14:57.281 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:14:57.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 msgr2=0x7fbd500a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 0x7fbd500a37d0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fbd58005950 tx=0x7fbd5800b410 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 msgr2=0x7fbd681999c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fbd5c003730 tx=0x7fbd5c00f690 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 shutdown_connections 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbd500a1310 0x7fbd500a37d0 secure :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fbd58005950 tx=0x7fbd5800b410 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbd6819dd10 0x7fbd6819e190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbd681031e0 0x7fbd68199480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 --2- 172.21.15.90:0/2959821121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbd68108300 0x7fbd681999c0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:57.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.280+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 >> 172.21.15.90:0/2959821121 conn(0x7fbd680fea40 msgr2=0x7fbd680ffe30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:57.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.282+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 shutdown_connections 2024-04-06T14:14:57.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:57.282+0000 7fbd6d3dc700 1 -- 172.21.15.90:0/2959821121 wait complete. 2024-04-06T14:14:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:57 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/513533807' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:57.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:57 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/513533807' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:14:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 550.0 s, 1625.2 tps, lat 61.203 ms stddev 65.684 2024-04-06T14:14:58.316 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:14:58.316 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:14:58.316 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- 172.21.15.90:0/223346144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2418100f50 msgr2=0x7f2418104a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 --2- 172.21.15.90:0/223346144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2418100f50 0x7f2418104a70 secure :-1 s=READY pgs=710 cs=0 l=1 rev1=1 crypto rx=0x7f2400009990 tx=0x7f2400009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- 172.21.15.90:0/223346144 shutdown_connections 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 --2- 172.21.15.90:0/223346144 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f241810cd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 --2- 172.21.15.90:0/223346144 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24181005a0 0x7f2418100980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 --2- 172.21.15.90:0/223346144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2418100f50 0x7f2418104a70 unknown :-1 s=CLOSED pgs=710 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- 172.21.15.90:0/223346144 >> 172.21.15.90:0/223346144 conn(0x7f24180761b0 msgr2=0x7f24180fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- 172.21.15.90:0/223346144 shutdown_connections 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- 172.21.15.90:0/223346144 wait complete. 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 Processor -- start 2024-04-06T14:14:58.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.472+0000 7f241e824700 1 -- start start 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f2418197940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f241819bce0 0x7f241819c160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f241819c760 con 0x7f24181005a0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f241819c8d0 con 0x7f241819bce0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f241819ca90 con 0x7f24181051a0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47970/0 (socket says 172.21.15.90:47970) 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 -- 172.21.15.90:0/2276345684 learned_addr learned my addr 172.21.15.90:0/2276345684 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241cdc1700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f241819bce0 0x7f241819c160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f240ffff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f2418197940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 -- 172.21.15.90:0/2276345684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 msgr2=0x7f2418197940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f2418197940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 -- 172.21.15.90:0/2276345684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f241819bce0 msgr2=0x7f241819c160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f241819bce0 0x7f241819c160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 -- 172.21.15.90:0/2276345684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24000095f0 con 0x7f24181005a0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f240ffff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f2418197940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2417fff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f240800e9a0 tx=0x7f240800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2415ffb700 1 -- 172.21.15.90:0/2276345684 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2408009790 con 0x7f24181005a0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2415ffb700 1 -- 172.21.15.90:0/2276345684 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f240800cbd0 con 0x7f24181005a0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f24181a3ff0 con 0x7f24181005a0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f2415ffb700 1 -- 172.21.15.90:0/2276345684 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2408018680 con 0x7f24181005a0 2024-04-06T14:14:58.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.473+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f24181a44c0 con 0x7f24181005a0 2024-04-06T14:14:58.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.476+0000 7f2415ffb700 1 -- 172.21.15.90:0/2276345684 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f240800cd40 con 0x7f24181005a0 2024-04-06T14:14:58.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.477+0000 7f2415ffb700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 0x7f24040a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.477+0000 7f2415ffb700 1 -- 172.21.15.90:0/2276345684 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2408014070 con 0x7f24181005a0 2024-04-06T14:14:58.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.479+0000 7f242a9d8b80 1 Processor -- start 2024-04-06T14:14:58.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.479+0000 7f242a9d8b80 1 -- start start 2024-04-06T14:14:58.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f240ffff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 0x7f24040a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba3920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba6250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49af1a10 con 0x563f49b99800 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49aef3d0 con 0x563f49ba3e60 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49aeacc0 con 0x563f49b9a120 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47978/0 (socket says 172.21.15.90:47978) 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 -- 172.21.15.90:0/1069884729 learned_addr learned my addr 172.21.15.90:0/1069884729 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f240ffff700 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f241cdc1700 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba6250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 -- 172.21.15.90:0/1069884729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 msgr2=0x563f49ba3920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba3920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 -- 172.21.15.90:0/1069884729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 msgr2=0x563f49ba6250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba6250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 -- 172.21.15.90:0/1069884729 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563f4996c430 con 0x563f49b99800 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f2417fff700 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f24080a1f30 tx=0x7f24080999c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=870e006ab444f45c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.480+0000 7f240ffff700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 0x7f24040a37d0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f2400006010 tx=0x7f240000b920 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.481+0000 7f240e7fc700 1 -- 172.21.15.90:0/1069884729 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f240809de10 con 0x563f49b99800 2024-04-06T14:14:58.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.481+0000 7f240e7fc700 1 -- 172.21.15.90:0/1069884729 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f24080a2460 con 0x563f49b99800 2024-04-06T14:14:58.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.481+0000 7f240e7fc700 1 -- 172.21.15.90:0/1069884729 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24080a1fd0 con 0x563f49b99800 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1069884729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 msgr2=0x563f49b99be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 secure :-1 s=READY pgs=712 cs=0 l=1 rev1=1 crypto rx=0x7f24080a1f30 tx=0x7f24080999c0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1069884729 shutdown_connections 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49b99be0 unknown :-1 s=CLOSED pgs=712 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1069884729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba3920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1069884729 >> 172.21.15.90:0/1069884729 conn(0x563f49af8720 msgr2=0x563f49b94b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1069884729 shutdown_connections 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1069884729 wait complete. 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 -- start start 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.482+0000 7f242a9d8b80 1 Processor -- start 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba0bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba1270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49a86e00 con 0x563f49b99800 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49a86aa0 con 0x563f49ba3e60 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f4996e720 con 0x563f49b9a120 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f242a9d8b80 10 client.0 _ll_register_callbacks cb 0x563f49a904f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47986/0 (socket says 172.21.15.90:47986) 2024-04-06T14:14:58.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 -- 172.21.15.90:0/1839062407 learned_addr learned my addr 172.21.15.90:0/1839062407 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f240ffff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba0bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f241cdc1700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba1270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 -- 172.21.15.90:0/1839062407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 msgr2=0x563f49ba0bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba0bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 -- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 msgr2=0x563f49ba1270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba1270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 -- 172.21.15.90:0/1839062407 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f24181a44c0 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f2417fff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f24080cd010 tx=0x7f2408099700 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d7744d804b4a9623 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.483+0000 7f23f77fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f240809de10 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563f49aab400 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f24080a21e0 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f24080b81c0 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 10 client.17055 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563f49918be0 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563f49aab400 con 0x563f49b99800 2024-04-06T14:14:58.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2408098fd0 con 0x563f49b99800 2024-04-06T14:14:58.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 10 client.17055 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:58.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 10 client.17055 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 20 client.17055 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49bb7b10 0x563f49bb9f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 4 client.17055 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.484+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563f49ae3780 con 0x563f49bb7b10 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.485+0000 7f240ffff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49bb7b10 0x563f49bb9f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.485+0000 7f23f77fe700 10 client.17055 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.485+0000 7f240ffff700 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49bb7b10 0x563f49bb9f00 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3dc9418bd8b85925 server_cookie=d26ed2e589f7cf2e in_seq=0 out_seq=0 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f23f77fe700 1 -- 172.21.15.90:0/1839062407 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563f49ae3780 con 0x563f49bb7b10 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f23f77fe700 10 client.17055 handle_command_reply: tid=0 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 client.17055 shutdown 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 msgr2=0x563f49ba3860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 secure :-1 s=READY pgs=713 cs=0 l=1 rev1=1 crypto rx=0x7f24080cd010 tx=0x7f2408099700 comp rx=0 tx=0).stop 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 shutdown_connections 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49ba3e60 0x563f49ba1270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b99800 0x563f49ba3860 unknown :-1 s=CLOSED pgs=713 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49bb7b10 0x563f49bb9f00 crc :-1 s=READY pgs=173 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1839062407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49ba0bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f23f77fe700 0 client.17055 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 >> 172.21.15.90:0/1839062407 conn(0x563f49af8720 msgr2=0x563f49b93ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 shutdown_connections 2024-04-06T14:14:58.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1839062407 wait complete. 2024-04-06T14:14:58.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.486+0000 7f242a9d8b80 20 client.17055 trim_cache size 0 max 16384 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.516+0000 7f242a9d8b80 1 Processor -- start 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 -- start start 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49b9f4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49ba3e60 0x563f49b94420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49b9ff40 con 0x563f49b9a120 2024-04-06T14:14:58.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49b95e40 con 0x563f49b99800 2024-04-06T14:14:58.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.522+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49b9b2a0 con 0x563f49ba3e60 2024-04-06T14:14:58.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.523+0000 7f240ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.524+0000 7f240ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:48000/0 (socket says 172.21.15.90:48000) 2024-04-06T14:14:58.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.524+0000 7f240ffff700 1 -- 172.21.15.90:0/620131176 learned_addr learned my addr 172.21.15.90:0/620131176 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.536+0000 7f240ffff700 1 -- 172.21.15.90:0/620131176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49ba3e60 msgr2=0x563f49b94420 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:58.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.536+0000 7f240ffff700 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49ba3e60 0x563f49b94420 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.536+0000 7f240ffff700 1 -- 172.21.15.90:0/620131176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 msgr2=0x563f49b9f4c0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:58.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.536+0000 7f240ffff700 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49b9f4c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.536+0000 7f240ffff700 1 -- 172.21.15.90:0/620131176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563f49b90be0 con 0x563f49b9a120 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.552+0000 7f240ffff700 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f2400015000 tx=0x7f2400026000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a9e29d84121ecf9b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.552+0000 7f240effd700 1 -- 172.21.15.90:0/620131176 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f240002e040 con 0x563f49b9a120 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.552+0000 7f240effd700 1 -- 172.21.15.90:0/620131176 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2400031550 con 0x563f49b9a120 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.552+0000 7f240effd700 1 -- 172.21.15.90:0/620131176 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2400053d50 con 0x563f49b9a120 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.553+0000 7f242a9d8b80 1 -- 172.21.15.90:0/620131176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 msgr2=0x563f49b9fa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.553+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 secure :-1 s=READY pgs=714 cs=0 l=1 rev1=1 crypto rx=0x7f2400015000 tx=0x7f2400026000 comp rx=0 tx=0).stop 2024-04-06T14:14:58.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.558+0000 7f242a9d8b80 1 -- 172.21.15.90:0/620131176 shutdown_connections 2024-04-06T14:14:58.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.558+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49ba3e60 0x563f49b94420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.558+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49b9f4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.558+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/620131176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49b9a120 0x563f49b9fa00 unknown :-1 s=CLOSED pgs=714 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.558+0000 7f242a9d8b80 1 -- 172.21.15.90:0/620131176 >> 172.21.15.90:0/620131176 conn(0x563f49af8720 msgr2=0x563f49baf170 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.562+0000 7f242a9d8b80 1 -- 172.21.15.90:0/620131176 shutdown_connections 2024-04-06T14:14:58.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.562+0000 7f242a9d8b80 1 -- 172.21.15.90:0/620131176 wait complete. 2024-04-06T14:14:58.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.562+0000 7f242a9d8b80 1 -- start start 2024-04-06T14:14:58.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.562+0000 7f242a9d8b80 1 Processor -- start 2024-04-06T14:14:58.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49ba9bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49baa100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49baa830 con 0x563f49ba3e60 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49c105f0 con 0x563f49b99800 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563f49c107b0 con 0x563f49b9a120 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f242a9d8b80 10 client.0 _ll_register_callbacks cb 0x563f49a904f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:48012/0 (socket says 172.21.15.90:48012) 2024-04-06T14:14:58.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 -- 172.21.15.90:0/1525969723 learned_addr learned my addr 172.21.15.90:0/1525969723 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 -- 172.21.15.90:0/1525969723 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 msgr2=0x563f49baa100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49baa100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 -- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 msgr2=0x563f49ba9bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49ba9bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.563+0000 7f241cdc1700 1 -- 172.21.15.90:0/1525969723 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2400020070 con 0x563f49ba3e60 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f241cdc1700 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f2410007b20 tx=0x7f2410007ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0d16eec88507c8c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f23f77fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:14:58.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2410014b40 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2410014ca0 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f24080a2460 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2410014e60 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f242a9d8b80 10 client.17061 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563f49ab0f20 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.564+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f24080a2460 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f24100214d0 con 0x563f49ba3e60 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 10 client.17061 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 10 client.17061 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 20 client.17061 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49c12050 0x563f49c14470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 4 client.17061 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x563f49939d70 con 0x563f49c12050 2024-04-06T14:14:58.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.565+0000 7f2417fff700 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49c12050 0x563f49c14470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.568+0000 7f23f77fe700 10 client.17061 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.568+0000 7f2417fff700 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49c12050 0x563f49c14470 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e2e64d80703ea1e0 server_cookie=2a48eb9189a03b3b in_seq=0 out_seq=0 2024-04-06T14:14:58.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.571+0000 7f23f77fe700 1 -- 172.21.15.90:0/1525969723 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x563f49939d70 con 0x563f49c12050 2024-04-06T14:14:58.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.571+0000 7f23f77fe700 10 client.17061 handle_command_reply: tid=0 2024-04-06T14:14:58.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 client.17061 shutdown 2024-04-06T14:14:58.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 msgr2=0x563f49c100b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 secure :-1 s=READY pgs=715 cs=0 l=1 rev1=1 crypto rx=0x7f2410007b20 tx=0x7f2410007ee0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 shutdown_connections 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563f49ba3e60 0x563f49c100b0 unknown :-1 s=CLOSED pgs=715 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563f49b99800 0x563f49ba9bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563f49c12050 0x563f49c14470 crc :-1 s=READY pgs=174 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 --2- 172.21.15.90:0/1525969723 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563f49b9a120 0x563f49baa100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f23f77fe700 0 client.17061 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 >> 172.21.15.90:0/1525969723 conn(0x563f49af8720 msgr2=0x563f49b9f480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 shutdown_connections 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 1 -- 172.21.15.90:0/1525969723 wait complete. 2024-04-06T14:14:58.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.572+0000 7f242a9d8b80 20 client.17061 trim_cache size 0 max 16384 2024-04-06T14:14:58.574 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:58.574 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:14:58.574 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "9f40b489-7cf2-4c70-a6c0-d3fc02f57a58", 2024-04-06T14:14:58.574 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:14:58.574 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 msgr2=0x7f24040a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 0x7f24040a37d0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f2400006010 tx=0x7f240000b920 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 msgr2=0x7f2418197400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 secure :-1 s=READY pgs=711 cs=0 l=1 rev1=1 crypto rx=0x7f240800e9a0 tx=0x7f240800ecb0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 shutdown_connections 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f24040a1310 0x7f24040a37d0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f241819bce0 0x7f241819c160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f24181005a0 0x7f2418197400 unknown :-1 s=CLOSED pgs=711 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 --2- 172.21.15.90:0/2276345684 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f24181051a0 0x7f2418197940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 >> 172.21.15.90:0/2276345684 conn(0x7f24180761b0 msgr2=0x7f24180ff500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 shutdown_connections 2024-04-06T14:14:58.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.573+0000 7f241e824700 1 -- 172.21.15.90:0/2276345684 wait complete. 2024-04-06T14:14:58.591 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:14:58.591 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 -- 172.21.15.90:0/2980085462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8100820 msgr2=0x7f5ed8104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2980085462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8100820 0x7f5ed8104840 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f5ec8009990 tx=0x7f5ec8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 -- 172.21.15.90:0/2980085462 shutdown_connections 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2980085462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8104f70 0x7f5ed810caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2980085462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5ed80ffe70 0x7f5ed8100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2980085462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8100820 0x7f5ed8104840 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 -- 172.21.15.90:0/2980085462 >> 172.21.15.90:0/2980085462 conn(0x7f5ed8076d50 msgr2=0x7f5ed8077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 -- 172.21.15.90:0/2980085462 shutdown_connections 2024-04-06T14:14:58.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.773+0000 7f5edf09b700 1 -- 172.21.15.90:0/2980085462 wait complete. 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 Processor -- start 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 -- start start 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5ed80ffe70 0x7f5ed8195eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8196980 0x7f5ed819af00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ed819b610 con 0x7f5ed80ffe70 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ed819b780 con 0x7f5ed8196980 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.774+0000 7f5edf09b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ed819b940 con 0x7f5ed8104f70 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57296/0 (socket says 172.21.15.90:57296) 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 -- 172.21.15.90:0/2945458313 learned_addr learned my addr 172.21.15.90:0/2945458313 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5edd638700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8196980 0x7f5ed819af00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 -- 172.21.15.90:0/2945458313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8196980 msgr2=0x7f5ed819af00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8196980 0x7f5ed819af00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 -- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5ed80ffe70 msgr2=0x7f5ed8195eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5ed80ffe70 0x7f5ed8195eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.775+0000 7f5ed7fff700 1 -- 172.21.15.90:0/2945458313 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ec80095f0 con 0x7f5ed8104f70 2024-04-06T14:14:58.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.778+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f5ec8000c00 tx=0x7f5ec8009510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.780+0000 7f5ed5ffb700 1 -- 172.21.15.90:0/2945458313 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ec801d070 con 0x7f5ed8104f70 2024-04-06T14:14:58.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.780+0000 7f5ed5ffb700 1 -- 172.21.15.90:0/2945458313 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ec8021410 con 0x7f5ed8104f70 2024-04-06T14:14:58.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.780+0000 7f5ed5ffb700 1 -- 172.21.15.90:0/2945458313 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ec8017650 con 0x7f5ed8104f70 2024-04-06T14:14:58.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.780+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ed819bbc0 con 0x7f5ed8104f70 2024-04-06T14:14:58.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.780+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ed819c020 con 0x7f5ed8104f70 2024-04-06T14:14:58.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.785+0000 7f5eeb24fb80 1 Processor -- start 2024-04-06T14:14:58.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.785+0000 7f5ed5ffb700 1 -- 172.21.15.90:0/2945458313 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5ec80217d0 con 0x7f5ed8104f70 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.785+0000 7f5eeb24fb80 1 -- start start 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5ed5ffb700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 0x7f5ec00a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce58a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce81d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42c33990 con 0x55ac42cdc0a0 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42c31350 con 0x55ac42cdb780 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42c2cc40 con 0x55ac42ce5de0 2024-04-06T14:14:58.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5ed5ffb700 1 -- 172.21.15.90:0/2945458313 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5ec80ce0a0 con 0x7f5ed8104f70 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34736/0 (socket says 172.21.15.90:34736) 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 -- 172.21.15.90:0/503877939 learned_addr learned my addr 172.21.15.90:0/503877939 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edd638700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 0x7f5ec00a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edd638700 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce81d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce58a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 -- 172.21.15.90:0/503877939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 msgr2=0x55ac42ce81d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce81d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 -- 172.21.15.90:0/503877939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 msgr2=0x55ac42ce58a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.786+0000 7f5edce37700 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce58a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5edce37700 1 -- 172.21.15.90:0/503877939 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ac42aae3b0 con 0x55ac42cdb780 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5edd638700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 0x7f5ec00a37d0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f5ed0015da0 tx=0x7f5ed0015420 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5edce37700 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc0098a0 tx=0x7f5ecc009bb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=234153079e5f7e21 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5ebeffd700 1 -- 172.21.15.90:0/503877939 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc011830 con 0x55ac42cdb780 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5ebeffd700 1 -- 172.21.15.90:0/503877939 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ecc011e50 con 0x55ac42cdb780 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.787+0000 7f5ebeffd700 1 -- 172.21.15.90:0/503877939 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc00b360 con 0x55ac42cdb780 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/503877939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 msgr2=0x55ac42cdbb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc0098a0 tx=0x7f5ecc009bb0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/503877939 shutdown_connections 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce81d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cdbb60 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/503877939 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce58a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/503877939 >> 172.21.15.90:0/503877939 conn(0x55ac42c3a6a0 msgr2=0x55ac42cd6a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/503877939 shutdown_connections 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.788+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/503877939 wait complete. 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 -- start start 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 Processor -- start 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce2b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42ce3250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42bc8d80 con 0x55ac42ce5de0 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42bc8a20 con 0x55ac42cdc0a0 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.789+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42ab06a0 con 0x55ac42cdb780 2024-04-06T14:14:58.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5eeb24fb80 10 client.0 _ll_register_callbacks cb 0x55ac42bd2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:57322/0 (socket says 172.21.15.90:57322) 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 -- 172.21.15.90:0/2530269035 learned_addr learned my addr 172.21.15.90:0/2530269035 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edd638700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42ce3250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 -- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdc0a0 msgr2=0x55ac42ce2b10 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce2b10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 -- 172.21.15.90:0/2530269035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 msgr2=0x55ac42ce3250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42ce3250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edce37700 1 -- 172.21.15.90:0/2530269035 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ecc0094d0 con 0x55ac42cdb780 2024-04-06T14:14:58.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.790+0000 7f5edd638700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42ce3250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5edce37700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc010910 tx=0x7f5ecc010430 comp rx=0 tx=0).ready entity=mon.1 client_cookie=53a9491d4f5a9771 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc010eb0 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ac42bed380 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5ecc011630 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5ecc0117f0 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 10 client.26542 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ac42a5ab60 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ac42bed380 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5ecc024c50 con 0x55ac42cdb780 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 10 client.26542 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 10 client.26542 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 20 client.26542 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:58.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d068d0 0x55ac42d08cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 4 client.26542 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.791+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ac42c25700 con 0x55ac42d068d0 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d068d0 0x55ac42d08cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5eb7fff700 10 client.26542 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d068d0 0x55ac42d08cc0 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c60e478a8ebfe92e server_cookie=cbc2fb30ace9cf04 in_seq=0 out_seq=0 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5eb7fff700 1 -- 172.21.15.90:0/2530269035 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ac42c25700 con 0x55ac42d068d0 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5eb7fff700 10 client.26542 handle_command_reply: tid=0 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.792+0000 7f5eeb24fb80 1 client.26542 shutdown 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.793+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 msgr2=0x55ac42ce5820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.793+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc010910 tx=0x7f5ecc010430 comp rx=0 tx=0).stop 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 shutdown_connections 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42ce3250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d068d0 0x55ac42d08cc0 crc :-1 s=READY pgs=175 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdb780 0x55ac42ce5820 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/2530269035 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdc0a0 0x55ac42ce2b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eb7fff700 0 client.26542 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 >> 172.21.15.90:0/2530269035 conn(0x55ac42c3a6a0 msgr2=0x55ac42cd5c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 shutdown_connections 2024-04-06T14:14:58.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.795+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/2530269035 wait complete. 2024-04-06T14:14:58.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.796+0000 7f5eeb24fb80 20 client.26542 trim_cache size 0 max 16384 2024-04-06T14:14:58.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.815+0000 7f5eeb24fb80 1 Processor -- start 2024-04-06T14:14:58.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.815+0000 7f5eeb24fb80 1 -- start start 2024-04-06T14:14:58.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfda70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce5350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42d02e10 con 0x55ac42cdc0a0 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42cde6d0 con 0x55ac42cdb780 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42ce81d0 con 0x55ac42ce5de0 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34766/0 (socket says 172.21.15.90:34766) 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 -- 172.21.15.90:0/4027042057 learned_addr learned my addr 172.21.15.90:0/4027042057 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edd638700 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce5350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 -- 172.21.15.90:0/4027042057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 msgr2=0x55ac42ce5350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce5350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 -- 172.21.15.90:0/4027042057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 msgr2=0x55ac42cfda70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfda70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 -- 172.21.15.90:0/4027042057 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ac42bf2ea0 con 0x55ac42cdb780 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edd638700 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce5350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.816+0000 7f5edce37700 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc009bf0 tx=0x7f5ecc010940 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3f947687d9929fd4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5ebf7fe700 1 -- 172.21.15.90:0/4027042057 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc01b6f0 con 0x55ac42cdb780 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5ebf7fe700 1 -- 172.21.15.90:0/4027042057 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ecc01bdf0 con 0x55ac42cdb780 2024-04-06T14:14:58.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5ebf7fe700 1 -- 172.21.15.90:0/4027042057 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc011bf0 con 0x55ac42cdb780 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/4027042057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 msgr2=0x55ac42ce2ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc009bf0 tx=0x7f5ecc010940 comp rx=0 tx=0).stop 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/4027042057 shutdown_connections 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42ce5de0 0x55ac42ce5350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42ce2ab0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/4027042057 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfda70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/4027042057 >> 172.21.15.90:0/4027042057 conn(0x55ac42c3a6a0 msgr2=0x55ac42ce23a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/4027042057 shutdown_connections 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.817+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/4027042057 wait complete. 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 -- start start 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 Processor -- start 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfbe40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42cfc380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42cfca90 con 0x55ac42ce5de0 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42cfcc00 con 0x55ac42cdb780 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ac42d5a700 con 0x55ac42cdc0a0 2024-04-06T14:14:58.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5eeb24fb80 10 client.0 _ll_register_callbacks cb 0x55ac42bd2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5edce37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34774/0 (socket says 172.21.15.90:34774) 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5edce37700 1 -- 172.21.15.90:0/1943838853 learned_addr learned my addr 172.21.15.90:0/1943838853 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.818+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfbe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edd638700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42cfc380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 -- 172.21.15.90:0/1943838853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 msgr2=0x55ac42cfbe40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfbe40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 -- 172.21.15.90:0/1943838853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 msgr2=0x55ac42cfc380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42cfc380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 -- 172.21.15.90:0/1943838853 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ac42a5ab60 con 0x55ac42cdb780 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfbe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:14:58.826 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5edce37700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc0048a0 tx=0x7f5ecc01b550 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aa0f26926920d97a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eb7fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ecc010690 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5ecc011e50 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(90..90 src has 1..90) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5ecc000e40 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5ecc011870 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eeb24fb80 10 client.26765 fetch_fsmap learned FSMap version 24 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ac42ab2cd0 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.819+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ecc011e50 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5ecc024c00 con 0x55ac42cdb780 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 10 client.26765 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:14:58.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 10 client.26765 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 20 client.26765 populate_metadata read hostname 'smithi090' 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d5b980 0x55ac42d5dd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 4 client.26765 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ac42a7bcf0 con 0x55ac42d5b980 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d5b980 0x55ac42d5dd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5ed7fff700 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d5b980 0x55ac42d5dd70 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2ed2d5f36c6e2637 server_cookie=f64402a5fd0784ba in_seq=0 out_seq=0 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eb7fff700 10 client.26765 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.820+0000 7f5eb7fff700 1 -- 172.21.15.90:0/1943838853 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55ac42a7bcf0 con 0x55ac42d5b980 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eb7fff700 10 client.26765 handle_command_reply: tid=0 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 client.26765 shutdown 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 msgr2=0x55ac42cfd9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f5ecc0048a0 tx=0x7f5ecc01b550 comp rx=0 tx=0).stop 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 shutdown_connections 2024-04-06T14:14:58.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ac42ce5de0 0x55ac42cfc380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ac42cdb780 0x55ac42cfd9c0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ac42d5b980 0x55ac42d5dd70 crc :-1 s=READY pgs=176 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 --2- 172.21.15.90:0/1943838853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ac42cdc0a0 0x55ac42cfbe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eb7fff700 0 client.26765 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 >> 172.21.15.90:0/1943838853 conn(0x55ac42c3a6a0 msgr2=0x55ac42cdb000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 shutdown_connections 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 1 -- 172.21.15.90:0/1943838853 wait complete. 2024-04-06T14:14:58.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.821+0000 7f5eeb24fb80 20 client.26765 trim_cache size 0 max 16384 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "9f40b489-7cf2-4c70-a6c0-d3fc02f57a58": { 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "9f40b489-7cf2-4c70-a6c0-d3fc02f57a58", 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:14:58.830 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:14:58.831 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:14:58.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 msgr2=0x7f5ec00a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 0x7f5ec00a37d0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f5ed0015da0 tx=0x7f5ed0015420 comp rx=0 tx=0).stop 2024-04-06T14:14:58.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 msgr2=0x7f5ed81963f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f5ec8000c00 tx=0x7f5ec8009510 comp rx=0 tx=0).stop 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 shutdown_connections 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5ec00a1310 0x7f5ec00a37d0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5ed8196980 0x7f5ed819af00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5ed80ffe70 0x7f5ed8195eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 --2- 172.21.15.90:0/2945458313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5ed8104f70 0x7f5ed81963f0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 >> 172.21.15.90:0/2945458313 conn(0x7f5ed8076d50 msgr2=0x7f5ed80fdeb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 shutdown_connections 2024-04-06T14:14:58.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:14:58.822+0000 7f5edf09b700 1 -- 172.21.15.90:0/2945458313 wait complete. 2024-04-06T14:14:58.836 INFO:tasks.cephfs.filesystem:scrub status for tag:9f40b489-7cf2-4c70-a6c0-d3fc02f57a58 - {'path': '/', 'tag': '9f40b489-7cf2-4c70-a6c0-d3fc02f57a58', 'options': 'recursive,force'} 2024-04-06T14:14:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:58 smithi090 ceph-mon[43420]: pgmap v296: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 9.7 MiB/s wr, 660 op/s 2024-04-06T14:14:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:58 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:14:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:58 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:59.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:58 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:14:59.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:58 smithi134 ceph-mon[47549]: pgmap v296: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 9.7 MiB/s wr, 660 op/s 2024-04-06T14:14:59.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:58 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:14:59.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:58 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:59.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:58 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:14:59.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:58 smithi105 ceph-mon[46969]: pgmap v296: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 9.7 MiB/s wr, 660 op/s 2024-04-06T14:14:59.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:58 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:14:59.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:58 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:14:59.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:58 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:15:00.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:14:59 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:00.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:14:59 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:00.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:14:59 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:01.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:00 smithi090 ceph-mon[43420]: pgmap v297: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 7.8 MiB/s wr, 555 op/s 2024-04-06T14:15:01.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:00 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:15:01.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:00 smithi134 ceph-mon[47549]: pgmap v297: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 7.8 MiB/s wr, 555 op/s 2024-04-06T14:15:01.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:00 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:15:01.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:00 smithi105 ceph-mon[46969]: pgmap v297: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 7.8 MiB/s wr, 555 op/s 2024-04-06T14:15:01.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:00 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:15:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 555.0 s, 1336.2 tps, lat 73.897 ms stddev 170.597 2024-04-06T14:15:03.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:02 smithi105 ceph-mon[46969]: pgmap v298: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 11 MiB/s wr, 828 op/s 2024-04-06T14:15:03.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:02 smithi090 ceph-mon[43420]: pgmap v298: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 11 MiB/s wr, 828 op/s 2024-04-06T14:15:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:02 smithi134 ceph-mon[47549]: pgmap v298: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 79 KiB/s rd, 11 MiB/s wr, 828 op/s 2024-04-06T14:15:04.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:03 smithi105 ceph-mon[46969]: pgmap v299: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 7.8 MiB/s wr, 625 op/s 2024-04-06T14:15:04.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:03 smithi090 ceph-mon[43420]: pgmap v299: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 7.8 MiB/s wr, 625 op/s 2024-04-06T14:15:04.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:03 smithi134 ceph-mon[47549]: pgmap v299: 97 pgs: 97 active+clean; 7.3 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 7.8 MiB/s wr, 625 op/s 2024-04-06T14:15:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:06 smithi134 ceph-mon[47549]: osdmap e91: 12 total, 12 up, 12 in 2024-04-06T14:15:06.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:06 smithi134 ceph-mon[47549]: pgmap v301: 97 pgs: 12 active+clean+snaptrim_wait, 79 active+clean, 6 active+clean+snaptrim; 7.4 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 23 MiB/s wr, 1.73k op/s 2024-04-06T14:15:06.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:06 smithi105 ceph-mon[46969]: osdmap e91: 12 total, 12 up, 12 in 2024-04-06T14:15:06.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:06 smithi105 ceph-mon[46969]: pgmap v301: 97 pgs: 12 active+clean+snaptrim_wait, 79 active+clean, 6 active+clean+snaptrim; 7.4 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 23 MiB/s wr, 1.73k op/s 2024-04-06T14:15:07.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:06 smithi090 ceph-mon[43420]: osdmap e91: 12 total, 12 up, 12 in 2024-04-06T14:15:07.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:06 smithi090 ceph-mon[43420]: pgmap v301: 97 pgs: 12 active+clean+snaptrim_wait, 79 active+clean, 6 active+clean+snaptrim; 7.4 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 23 MiB/s wr, 1.73k op/s 2024-04-06T14:15:08.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 560.0 s, 961.8 tps, lat 105.260 ms stddev 148.319 2024-04-06T14:15:09.153 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:08 smithi134 ceph-mon[47549]: pgmap v302: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:09.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:08 smithi105 ceph-mon[46969]: pgmap v302: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:09.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:08 smithi090 ceph-mon[43420]: pgmap v302: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:10.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:10 smithi090 ceph-mon[43420]: pgmap v303: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:10.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:10 smithi134 ceph-mon[47549]: pgmap v303: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:10.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:10 smithi105 ceph-mon[46969]: pgmap v303: 97 pgs: 18 active+clean+snaptrim_wait, 69 active+clean, 10 active+clean+snaptrim; 7.2 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 108 KiB/s rd, 30 MiB/s wr, 2.19k op/s 2024-04-06T14:15:13.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 565.0 s, 1436.6 tps, lat 69.424 ms stddev 79.471 2024-04-06T14:15:13.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:12 smithi105 ceph-mon[46969]: pgmap v304: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 32 MiB/s wr, 2.19k op/s 2024-04-06T14:15:13.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:12 smithi105 ceph-mon[46969]: osdmap e92: 12 total, 12 up, 12 in 2024-04-06T14:15:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:12 smithi090 ceph-mon[43420]: pgmap v304: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 32 MiB/s wr, 2.19k op/s 2024-04-06T14:15:13.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:12 smithi090 ceph-mon[43420]: osdmap e92: 12 total, 12 up, 12 in 2024-04-06T14:15:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:12 smithi134 ceph-mon[47549]: pgmap v304: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 32 MiB/s wr, 2.19k op/s 2024-04-06T14:15:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:12 smithi134 ceph-mon[47549]: osdmap e92: 12 total, 12 up, 12 in 2024-04-06T14:15:15.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:14 smithi105 ceph-mon[46969]: pgmap v306: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 24 MiB/s wr, 1.60k op/s 2024-04-06T14:15:15.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:14 smithi090 ceph-mon[43420]: pgmap v306: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 24 MiB/s wr, 1.60k op/s 2024-04-06T14:15:15.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:14 smithi134 ceph-mon[47549]: pgmap v306: 97 pgs: 97 active+clean; 6.3 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 24 MiB/s wr, 1.60k op/s 2024-04-06T14:15:17.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:16 smithi105 ceph-mon[46969]: pgmap v307: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 21 MiB/s wr, 1.17k op/s 2024-04-06T14:15:17.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:16 smithi090 ceph-mon[43420]: pgmap v307: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 21 MiB/s wr, 1.17k op/s 2024-04-06T14:15:17.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:16 smithi134 ceph-mon[47549]: pgmap v307: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 21 MiB/s wr, 1.17k op/s 2024-04-06T14:15:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 570.0 s, 1632.0 tps, lat 61.588 ms stddev 67.268 2024-04-06T14:15:19.039 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:18 smithi090 ceph-mon[43420]: pgmap v308: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:19.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:18 smithi134 ceph-mon[47549]: pgmap v308: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:19.252 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:18 smithi105 ceph-mon[46969]: pgmap v308: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:21.252 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:20 smithi105 ceph-mon[46969]: pgmap v309: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:20 smithi090 ceph-mon[43420]: pgmap v309: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:21.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:20 smithi134 ceph-mon[47549]: pgmap v309: 97 pgs: 97 active+clean; 6.4 GiB data, 16 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 721 op/s 2024-04-06T14:15:23.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 575.0 s, 1610.0 tps, lat 62.376 ms stddev 70.622 2024-04-06T14:15:23.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:22 smithi105 ceph-mon[46969]: pgmap v310: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 16 MiB/s wr, 665 op/s 2024-04-06T14:15:23.297 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:22 smithi090 ceph-mon[43420]: pgmap v310: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 16 MiB/s wr, 665 op/s 2024-04-06T14:15:23.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:22 smithi134 ceph-mon[47549]: pgmap v310: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 16 MiB/s wr, 665 op/s 2024-04-06T14:15:25.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:24 smithi105 ceph-mon[46969]: pgmap v311: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 14 MiB/s wr, 595 op/s 2024-04-06T14:15:25.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:24 smithi090 ceph-mon[43420]: pgmap v311: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 14 MiB/s wr, 595 op/s 2024-04-06T14:15:25.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:24 smithi134 ceph-mon[47549]: pgmap v311: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 14 MiB/s wr, 595 op/s 2024-04-06T14:15:26.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:26 smithi134 ceph-mon[47549]: pgmap v312: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 770 op/s 2024-04-06T14:15:26.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:26 smithi105 ceph-mon[46969]: pgmap v312: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 770 op/s 2024-04-06T14:15:26.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:26 smithi090 ceph-mon[43420]: pgmap v312: 97 pgs: 97 active+clean; 6.5 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 770 op/s 2024-04-06T14:15:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 580.0 s, 950.6 tps, lat 104.882 ms stddev 155.036 2024-04-06T14:15:28.837 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:15:28.837 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 -- 172.21.15.90:0/3986939722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 msgr2=0x7fa428101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 --2- 172.21.15.90:0/3986939722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 0x7fa428101370 secure :-1 s=READY pgs=716 cs=0 l=1 rev1=1 crypto rx=0x7fa418009990 tx=0x7fa418009ca0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 -- 172.21.15.90:0/3986939722 shutdown_connections 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 --2- 172.21.15.90:0/3986939722 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 0x7fa42810dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 --2- 172.21.15.90:0/3986939722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 0x7fa428101370 unknown :-1 s=CLOSED pgs=716 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 --2- 172.21.15.90:0/3986939722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa428101940 0x7fa428105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 -- 172.21.15.90:0/3986939722 >> 172.21.15.90:0/3986939722 conn(0x7fa4280fc7f0 msgr2=0x7fa4280fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 -- 172.21.15.90:0/3986939722 shutdown_connections 2024-04-06T14:15:29.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 -- 172.21.15.90:0/3986939722 wait complete. 2024-04-06T14:15:29.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.010+0000 7fa42d305700 1 Processor -- start 2024-04-06T14:15:29.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 -- start start 2024-04-06T14:15:29.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 0x7fa4281a35e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 0x7fa4281a3b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa42810d700 con 0x7fa428100f90 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa42810d870 con 0x7fa42810cc40 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.011+0000 7fa42d305700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa42810da30 con 0x7fa4281060b0 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43462/0 (socket says 172.21.15.90:43462) 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 -- 172.21.15.90:0/336390554 learned_addr learned my addr 172.21.15.90:0/336390554 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4267fc700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 0x7fa4281a3b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 -- 172.21.15.90:0/336390554 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 msgr2=0x7fa4281a3b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 0x7fa4281a3b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 -- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 msgr2=0x7fa4281a35e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 0x7fa4281a35e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 -- 172.21.15.90:0/336390554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa4180095f0 con 0x7fa42810cc40 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa4277fe700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fa420007ee0 tx=0x7fa42000d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa40ffff700 1 -- 172.21.15.90:0/336390554 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa420017070 con 0x7fa42810cc40 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa40ffff700 1 -- 172.21.15.90:0/336390554 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa42000f040 con 0x7fa42810cc40 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa40ffff700 1 -- 172.21.15.90:0/336390554 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa4200057d0 con 0x7fa42810cc40 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.012+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa42806a890 con 0x7fa42810cc40 2024-04-06T14:15:29.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.013+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa42806ad20 con 0x7fa42810cc40 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.016+0000 7fa4394b9b80 1 Processor -- start 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.016+0000 7fa4394b9b80 1 -- start start 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.016+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756ba780 0x55e6756bab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.016+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c48a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 0x55e6756c71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e675612990 con 0x55e6756bb0a0 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e675610350 con 0x55e6756c4de0 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e67560bc40 con 0x55e6756ba780 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa426ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c48a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa426ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c48a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54702/0 (socket says 172.21.15.90:54702) 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa426ffd700 1 -- 172.21.15.90:0/438313736 learned_addr learned my addr 172.21.15.90:0/438313736 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756ba780 0x55e6756bab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 0x55e6756c71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 -- 172.21.15.90:0/438313736 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756ba780 msgr2=0x55e6756bab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756ba780 0x55e6756bab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 -- 172.21.15.90:0/438313736 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 msgr2=0x55e6756c48a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c48a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 -- 172.21.15.90:0/438313736 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e67548d3b0 con 0x55e6756c4de0 2024-04-06T14:15:29.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.017+0000 7fa4267fc700 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 0x55e6756c71d0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fa42819ef10 tx=0x7fa41c009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ea1d32869d4f30a8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40ffff700 1 -- 172.21.15.90:0/336390554 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa420005930 con 0x7fa42810cc40 2024-04-06T14:15:29.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40d7fa700 1 -- 172.21.15.90:0/438313736 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa41c012050 con 0x55e6756c4de0 2024-04-06T14:15:29.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40d7fa700 1 -- 172.21.15.90:0/438313736 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa41c0044f0 con 0x55e6756c4de0 2024-04-06T14:15:29.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40d7fa700 1 -- 172.21.15.90:0/438313736 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa41c003c20 con 0x55e6756c4de0 2024-04-06T14:15:29.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40ffff700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 0x7fa4100c57e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.018+0000 7fa40ffff700 1 -- 172.21.15.90:0/336390554 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa4200cc490 con 0x7fa42810cc40 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 -- 172.21.15.90:0/438313736 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 msgr2=0x55e6756c71d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 0x55e6756c71d0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7fa42819ef10 tx=0x7fa41c009260 comp rx=0 tx=0).stop 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 -- 172.21.15.90:0/438313736 shutdown_connections 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756c4de0 0x55e6756c71d0 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756ba780 0x55e6756bab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/438313736 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c48a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa4394b9b80 1 -- 172.21.15.90:0/438313736 >> 172.21.15.90:0/438313736 conn(0x55e6756196a0 msgr2=0x55e6756b5a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.019+0000 7fa426ffd700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 0x7fa4100c57e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.020+0000 7fa4394b9b80 1 -- 172.21.15.90:0/438313736 shutdown_connections 2024-04-06T14:15:29.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.020+0000 7fa4394b9b80 1 -- 172.21.15.90:0/438313736 wait complete. 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.020+0000 7fa426ffd700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 0x7fa4100c57e0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fa418005b40 tx=0x7fa418005a90 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.020+0000 7fa4394b9b80 1 -- start start 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 Processor -- start 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c1bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756c22f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6755a7d80 con 0x55e6756bb0a0 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6755a7a20 con 0x55e6756ba780 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e67548f6a0 con 0x55e6756c4de0 2024-04-06T14:15:29.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4394b9b80 10 client.0 _ll_register_callbacks cb 0x55e6755b1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa426ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa426ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43478/0 (socket says 172.21.15.90:43478) 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa426ffd700 1 -- 172.21.15.90:0/2698172576 learned_addr learned my addr 172.21.15.90:0/2698172576 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.021+0000 7fa4277fe700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756c22f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 -- 172.21.15.90:0/2698172576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 msgr2=0x55e6756c22f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756c22f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 -- 172.21.15.90:0/2698172576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 msgr2=0x55e6756c1bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c1bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 -- 172.21.15.90:0/2698172576 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa41c01a040 con 0x55e6756ba780 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fa4180053d0 tx=0x7fa418009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1a488061b2a79ef1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa40aff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa418004a10 con 0x55e6756ba780 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa418004b70 con 0x55e6756ba780 2024-04-06T14:15:29.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e6755cc380 con 0x55e6756ba780 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa418004d30 con 0x55e6756ba780 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa4394b9b80 10 client.26777 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.022+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e675439b60 con 0x55e6756ba780 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa41803b080 con 0x55e6756ba780 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa418037540 con 0x55e6756ba780 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 10 client.26777 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 10 client.26777 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 20 client.26777 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e6756d4ab0 0x55e6756d6ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 4 client.26777 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:29.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e675604700 con 0x55e6756d4ab0 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.023+0000 7fa4267fc700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e6756d4ab0 0x55e6756d6ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.024+0000 7fa4267fc700 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e6756d4ab0 0x55e6756d6ed0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a3afc073c04b70fa server_cookie=641e6126fc80b937 in_seq=0 out_seq=0 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.024+0000 7fa40aff5700 10 client.26777 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.024+0000 7fa40aff5700 1 -- 172.21.15.90:0/2698172576 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55e675604700 con 0x55e6756d4ab0 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.024+0000 7fa40aff5700 10 client.26777 handle_command_reply: tid=0 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.024+0000 7fa4394b9b80 1 client.26777 shutdown 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 msgr2=0x55e6756c4870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fa4180053d0 tx=0x7fa418009310 comp rx=0 tx=0).stop 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 shutdown_connections 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e6756d4ab0 0x55e6756d6ed0 crc :-1 s=READY pgs=177 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756c22f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa40aff5700 0 client.26777 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:29.028 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756c4870 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2698172576 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756c1bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 >> 172.21.15.90:0/2698172576 conn(0x55e6756196a0 msgr2=0x55e6756b4c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 shutdown_connections 2024-04-06T14:15:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2698172576 wait complete. 2024-04-06T14:15:29.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.025+0000 7fa4394b9b80 20 client.26777 trim_cache size 0 max 16384 2024-04-06T14:15:29.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:28 smithi090 ceph-mon[43420]: pgmap v313: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 16 MiB/s wr, 945 op/s 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 Processor -- start 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 -- start start 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cba00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756b58e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756cbf40 con 0x55e6756bb0a0 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756ad3c0 con 0x55e6756ba780 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756b71f0 con 0x55e6756c4de0 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.047+0000 7fa4267fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cba00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa4267fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54720/0 (socket says 172.21.15.90:54720) 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa4267fc700 1 -- 172.21.15.90:0/1200483976 learned_addr learned my addr 172.21.15.90:0/1200483976 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa4277fe700 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756b58e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 -- 172.21.15.90:0/1200483976 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 msgr2=0x55e6756b58e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756b58e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 -- 172.21.15.90:0/1200483976 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 msgr2=0x55e6756b53a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 -- 172.21.15.90:0/1200483976 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e6755d1ea0 con 0x55e6756ba780 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa4267fc700 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:29.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa426ffd700 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cba00 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa4180053a0 tx=0x7fa418004500 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a869b848c7f2541 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa40dffb700 1 -- 172.21.15.90:0/1200483976 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa418037ac0 con 0x55e6756ba780 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa40dffb700 1 -- 172.21.15.90:0/1200483976 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e6755cc380 con 0x55e6756ba780 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.048+0000 7fa40dffb700 1 -- 172.21.15.90:0/1200483976 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa41800fa20 con 0x55e6756ba780 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- 172.21.15.90:0/1200483976 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 msgr2=0x55e6756cba00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cba00 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7fa4180053a0 tx=0x7fa418004500 comp rx=0 tx=0).stop 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- 172.21.15.90:0/1200483976 shutdown_connections 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756c4de0 0x55e6756b58e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cba00 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/1200483976 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756bb0a0 0x55e6756b53a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- 172.21.15.90:0/1200483976 >> 172.21.15.90:0/1200483976 conn(0x55e6756196a0 msgr2=0x55e6756c0480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.054 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- 172.21.15.90:0/1200483976 shutdown_connections 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- 172.21.15.90:0/1200483976 wait complete. 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 -- start start 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.049+0000 7fa4394b9b80 1 Processor -- start 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cb8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756bb0a0 0x55e6756c9df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756ca8b0 con 0x55e6756c4de0 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756caa20 con 0x55e6756ba780 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e6756cabe0 con 0x55e6756bb0a0 2024-04-06T14:15:29.055 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4394b9b80 10 client.0 _ll_register_callbacks cb 0x55e6755b1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4277fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4277fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54736/0 (socket says 172.21.15.90:54736) 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4277fe700 1 -- 172.21.15.90:0/2547382222 learned_addr learned my addr 172.21.15.90:0/2547382222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cb8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4267fc700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756bb0a0 0x55e6756c9df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 -- 172.21.15.90:0/2547382222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756bb0a0 msgr2=0x55e6756c9df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756bb0a0 0x55e6756c9df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 -- 172.21.15.90:0/2547382222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 msgr2=0x55e6756ca330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa426ffd700 1 -- 172.21.15.90:0/2547382222 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55e67548d3b0 con 0x55e6756ba780 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.050+0000 7fa4277fe700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa426ffd700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cb8a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fa41800b690 tx=0x7fa418038f30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8893551edf27cf99 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa40aff5700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa418004dd0 con 0x55e6756ba780 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa41800f460 con 0x55e6756ba780 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa41c0044f0 con 0x55e6756ba780 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa41800f620 con 0x55e6756ba780 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa4394b9b80 10 client.26783 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:29.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.051+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e675439b60 con 0x55e6756ba780 2024-04-06T14:15:29.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.052+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa41c0044f0 con 0x55e6756ba780 2024-04-06T14:15:29.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.052+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa418041790 con 0x55e6756ba780 2024-04-06T14:15:29.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 10 client.26783 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:29.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 10 client.26783 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 20 client.26783 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e675729990 0x55e67572bd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 4 client.26783 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55e67545acf0 con 0x55e675729990 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.053+0000 7fa4267fc700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e675729990 0x55e67572bd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa40aff5700 10 client.26783 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4267fc700 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e675729990 0x55e67572bd80 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=60f4e12c5604466a server_cookie=175931ade434bfe6 in_seq=0 out_seq=0 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa40aff5700 1 -- 172.21.15.90:0/2547382222 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55e67545acf0 con 0x55e675729990 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa40aff5700 10 client.26783 handle_command_reply: tid=0 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 client.26783 shutdown 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 msgr2=0x55e6756cb8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cb8a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fa41800b690 tx=0x7fa418038f30 comp rx=0 tx=0).stop 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 shutdown_connections 2024-04-06T14:15:29.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e6756c4de0 0x55e6756ca330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e6756ba780 0x55e6756cb8a0 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55e675729990 0x55e67572bd80 crc :-1 s=READY pgs=178 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 --2- 172.21.15.90:0/2547382222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e6756bb0a0 0x55e6756c9df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa40aff5700 0 client.26783 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 >> 172.21.15.90:0/2547382222 conn(0x55e6756196a0 msgr2=0x55e6756c1980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 shutdown_connections 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 1 -- 172.21.15.90:0/2547382222 wait complete. 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.054+0000 7fa4394b9b80 20 client.26783 trim_cache size 0 max 16384 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:15:29.061 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:15:29.062 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:15:29.062 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:15:29.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 msgr2=0x7fa4100c57e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 0x7fa4100c57e0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fa418005b40 tx=0x7fa418005a90 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 msgr2=0x7fa42810d0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fa420007ee0 tx=0x7fa42000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 shutdown_connections 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4100c3320 0x7fa4100c57e0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa42810cc40 0x7fa42810d0c0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa428100f90 0x7fa4281a35e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 --2- 172.21.15.90:0/336390554 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4281060b0 0x7fa4281a3b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 >> 172.21.15.90:0/336390554 conn(0x7fa4280fc7f0 msgr2=0x7fa4280fdae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 shutdown_connections 2024-04-06T14:15:29.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.055+0000 7fa42d305700 1 -- 172.21.15.90:0/336390554 wait complete. 2024-04-06T14:15:29.071 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:15:29.071 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:15:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:28 smithi134 ceph-mon[47549]: pgmap v313: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 16 MiB/s wr, 945 op/s 2024-04-06T14:15:29.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:28 smithi105 ceph-mon[46969]: pgmap v313: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 16 MiB/s wr, 945 op/s 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 -- 172.21.15.90:0/1166044416 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc0ff3b0 msgr2=0x7fdccc0ff790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 --2- 172.21.15.90:0/1166044416 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc0ff790 secure :-1 s=READY pgs=717 cs=0 l=1 rev1=1 crypto rx=0x7fdcbc009990 tx=0x7fdcbc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 -- 172.21.15.90:0/1166044416 shutdown_connections 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 --2- 172.21.15.90:0/1166044416 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc105f70 0x7fdccc10db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 --2- 172.21.15.90:0/1166044416 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc0ff790 unknown :-1 s=CLOSED pgs=717 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 --2- 172.21.15.90:0/1166044416 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc0ffd60 0x7fdccc105890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 -- 172.21.15.90:0/1166044416 >> 172.21.15.90:0/1166044416 conn(0x7fdccc0760a0 msgr2=0x7fdccc0fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 -- 172.21.15.90:0/1166044416 shutdown_connections 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 -- 172.21.15.90:0/1166044416 wait complete. 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.247+0000 7fdcd2125700 1 Processor -- start 2024-04-06T14:15:29.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 -- start start 2024-04-06T14:15:29.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc100320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 0x7fdccc100860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdccc1a5900 con 0x7fdccc100da0 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdccc1a5a70 con 0x7fdccc105f70 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdcd2125700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdccc1a5c30 con 0x7fdccc0ff3b0 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54744/0 (socket says 172.21.15.90:54744) 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 -- 172.21.15.90:0/2621428250 learned_addr learned my addr 172.21.15.90:0/2621428250 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccaffd700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 0x7fdccc100860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccb7fe700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc100320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 -- 172.21.15.90:0/2621428250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc0ff3b0 msgr2=0x7fdccc100320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc100320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 -- 172.21.15.90:0/2621428250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 msgr2=0x7fdccc100860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 0x7fdccc100860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccbfff700 1 -- 172.21.15.90:0/2621428250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdcc0009520 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.248+0000 7fdccaffd700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 0x7fdccc100860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.249+0000 7fdccbfff700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7fdcc4007ee0 tx=0x7fdcc400d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.249+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdcc4017070 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.249+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdcbc0095f0 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.249+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdccc1a6290 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.250+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdcc400f040 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.250+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdccc109220 con 0x7fdccc100da0 2024-04-06T14:15:29.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.250+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdcc4005580 con 0x7fdccc100da0 2024-04-06T14:15:29.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.253+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fdcc4004ad0 con 0x7fdccc100da0 2024-04-06T14:15:29.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.254+0000 7fdcc8ff9700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 0x7fdcb40a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.254+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fdcc40ccde0 con 0x7fdccc100da0 2024-04-06T14:15:29.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.255+0000 7fdccb7fe700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 0x7fdcb40a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.255+0000 7fdccb7fe700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 0x7fdcb40a37d0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fdcbc00b2e0 tx=0x7fdcbc005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.259+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fdcc4096210 con 0x7fdccc100da0 2024-04-06T14:15:29.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.569+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fdccc1a5df0 con 0x7fdccc100da0 2024-04-06T14:15:29.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.569+0000 7fdcc8ff9700 1 -- 172.21.15.90:0/2621428250 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fdcc4099830 con 0x7fdccc100da0 2024-04-06T14:15:29.571 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:15:29.571 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:15:29.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 msgr2=0x7fdcb40a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 0x7fdcb40a37d0 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7fdcbc00b2e0 tx=0x7fdcbc005fd0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 msgr2=0x7fdccc1a5330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 secure :-1 s=READY pgs=718 cs=0 l=1 rev1=1 crypto rx=0x7fdcc4007ee0 tx=0x7fdcc400d3b0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 shutdown_connections 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdcb40a1310 0x7fdcb40a37d0 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdccc100da0 0x7fdccc1a5330 unknown :-1 s=CLOSED pgs=718 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdccc0ff3b0 0x7fdccc100320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 --2- 172.21.15.90:0/2621428250 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdccc105f70 0x7fdccc100860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 >> 172.21.15.90:0/2621428250 conn(0x7fdccc0760a0 msgr2=0x7fdccc1042a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 shutdown_connections 2024-04-06T14:15:29.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.573+0000 7fdcd2125700 1 -- 172.21.15.90:0/2621428250 wait complete. 2024-04-06T14:15:29.575 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:15:29.589 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:15:29.589 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 -- 172.21.15.90:0/1551875353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c104f80 msgr2=0x7fa66c10cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 --2- 172.21.15.90:0/1551875353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c104f80 0x7fa66c10cb00 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fa664009f90 tx=0x7fa66400b330 comp rx=0 tx=0).stop 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 -- 172.21.15.90:0/1551875353 shutdown_connections 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 --2- 172.21.15.90:0/1551875353 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c104f80 0x7fa66c10cb00 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 --2- 172.21.15.90:0/1551875353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa66c0ffe60 0x7fa66c100240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 --2- 172.21.15.90:0/1551875353 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c100810 0x7fa66c104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 -- 172.21.15.90:0/1551875353 >> 172.21.15.90:0/1551875353 conn(0x7fa66c076db0 msgr2=0x7fa66c0771c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 -- 172.21.15.90:0/1551875353 shutdown_connections 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.765+0000 7fa673364700 1 -- 172.21.15.90:0/1551875353 wait complete. 2024-04-06T14:15:29.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 Processor -- start 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 -- start start 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa66c104f80 0x7fa66c195650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 0x7fa66c199e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66c19a470 con 0x7fa66c1999f0 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66c19a5e0 con 0x7fa66c104f80 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa673364700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa66c19a7a0 con 0x7fa66c0ffe60 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa671100700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa671901700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 0x7fa66c199e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa671100700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58760/0 (socket says 172.21.15.90:58760) 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.766+0000 7fa671100700 1 -- 172.21.15.90:0/3385430647 learned_addr learned my addr 172.21.15.90:0/3385430647 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 -- 172.21.15.90:0/3385430647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa66c104f80 msgr2=0x7fa66c195650 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:15:29.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa66c104f80 0x7fa66c195650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 -- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 msgr2=0x7fa66c199e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 0x7fa66c199e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 -- 172.21.15.90:0/3385430647 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa668009520 con 0x7fa66c0ffe60 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671901700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 0x7fa66c199e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa671100700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fa66800eb30 tx=0x7fa66800ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa6627fc700 1 -- 172.21.15.90:0/3385430647 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa668009880 con 0x7fa66c0ffe60 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa6627fc700 1 -- 172.21.15.90:0/3385430647 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa6680044e0 con 0x7fa66c0ffe60 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa65c0095f0 con 0x7fa66c0ffe60 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa6627fc700 1 -- 172.21.15.90:0/3385430647 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa668003e90 con 0x7fa66c0ffe60 2024-04-06T14:15:29.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.767+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa664009c40 con 0x7fa66c0ffe60 2024-04-06T14:15:29.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.770+0000 7fa6627fc700 1 -- 172.21.15.90:0/3385430647 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa668004000 con 0x7fa66c0ffe60 2024-04-06T14:15:29.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.771+0000 7fa6627fc700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 0x7fa6580a59a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.771+0000 7fa6708ff700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 0x7fa6580a59a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.771+0000 7fa6627fc700 1 -- 172.21.15.90:0/3385430647 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa668014070 con 0x7fa66c0ffe60 2024-04-06T14:15:29.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.772+0000 7fa67f518b80 1 Processor -- start 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.772+0000 7fa67f518b80 1 -- start start 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c79780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c7a140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c8e870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27bd1990 con 0x555d27c8c480 2024-04-06T14:15:29.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27bcf350 con 0x555d27c79cc0 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa67f518b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27bcac40 con 0x555d27c7b380 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa6708ff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c7a140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa6708ff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c7a140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43544/0 (socket says 172.21.15.90:43544) 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa6708ff700 1 -- 172.21.15.90:0/463384215 learned_addr learned my addr 172.21.15.90:0/463384215 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c79780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 -- 172.21.15.90:0/463384215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 msgr2=0x555d27c7a140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c7a140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 -- 172.21.15.90:0/463384215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 msgr2=0x555d27c8e870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c8e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.773+0000 7fa671100700 1 -- 172.21.15.90:0/463384215 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555d27a4c3b0 con 0x555d27c7b380 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.774+0000 7fa671100700 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c79780 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fa6680ca8b0 tx=0x7fa668099db0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=43907ec878a68b5c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.774+0000 7fa6708ff700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 0x7fa6580a59a0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fa66c196660 tx=0x7fa65c009310 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.774+0000 7fa6577fe700 1 -- 172.21.15.90:0/463384215 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa66800cc90 con 0x555d27c7b380 2024-04-06T14:15:29.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.774+0000 7fa6577fe700 1 -- 172.21.15.90:0/463384215 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa6680af070 con 0x555d27c7b380 2024-04-06T14:15:29.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.774+0000 7fa6577fe700 1 -- 172.21.15.90:0/463384215 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa668098610 con 0x555d27c7b380 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- 172.21.15.90:0/463384215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 msgr2=0x555d27c79780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c79780 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fa6680ca8b0 tx=0x7fa668099db0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- 172.21.15.90:0/463384215 shutdown_connections 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c8e870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c79780 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 --2- 172.21.15.90:0/463384215 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c7a140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- 172.21.15.90:0/463384215 >> 172.21.15.90:0/463384215 conn(0x555d27bd86a0 msgr2=0x555d27c74a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- 172.21.15.90:0/463384215 shutdown_connections 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- 172.21.15.90:0/463384215 wait complete. 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 -- start start 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.775+0000 7fa67f518b80 1 Processor -- start 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.776+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.776+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 0x555d27c86130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa6708ff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 0x555d27c86130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa6708ff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 0x555d27c86130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54800/0 (socket says 172.21.15.90:54800) 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43550/0 (socket says 172.21.15.90:43550) 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c8c480 0x555d27c8b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa67f518b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27b66d80 con 0x555d27c7b380 2024-04-06T14:15:29.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa67f518b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27b66a20 con 0x555d27c79cc0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa67f518b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27a4e6a0 con 0x555d27c8c480 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa67f518b80 10 client.0 _ll_register_callbacks cb 0x555d27b70470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa6708ff700 1 -- 172.21.15.90:0/3262273186 learned_addr learned my addr 172.21.15.90:0/3262273186 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671901700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c8c480 0x555d27c8b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 -- 172.21.15.90:0/3262273186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c8c480 msgr2=0x555d27c8b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c8c480 0x555d27c8b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 -- 172.21.15.90:0/3262273186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 msgr2=0x555d27c86130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 0x555d27c86130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 -- 172.21.15.90:0/3262273186 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa664009c40 con 0x555d27c79cc0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa671100700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fa6680ca8e0 tx=0x7fa66809aaa0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=265b931bf2c20eb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.777+0000 7fa654ff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa66809a270 con 0x555d27c79cc0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa66809e0a0 con 0x555d27c79cc0 2024-04-06T14:15:29.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555d27b8b380 con 0x555d27c79cc0 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa6680a26f0 con 0x555d27c79cc0 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 10 client.26798 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555d279f8b60 con 0x555d27c79cc0 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555d27b8b380 con 0x555d27c79cc0 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa668099760 con 0x555d27c79cc0 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 10 client.26798 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 10 client.26798 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 20 client.26798 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27ca4710 0x555d27ca6b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 4 client.26798 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555d27bc3700 con 0x555d27ca4710 2024-04-06T14:15:29.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.778+0000 7fa6708ff700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27ca4710 0x555d27ca6b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.779+0000 7fa6708ff700 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27ca4710 0x555d27ca6b00 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e364aee2c09c305b server_cookie=100fe95b49039914 in_seq=0 out_seq=0 2024-04-06T14:15:29.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.779+0000 7fa654ff9700 10 client.26798 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.779+0000 7fa654ff9700 1 -- 172.21.15.90:0/3262273186 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555d27bc3700 con 0x555d27ca4710 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.779+0000 7fa654ff9700 10 client.26798 handle_command_reply: tid=0 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.779+0000 7fa67f518b80 1 client.26798 shutdown 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 msgr2=0x555d27c85bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fa6680ca8e0 tx=0x7fa66809aaa0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 shutdown_connections 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27ca4710 0x555d27ca6b00 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c8c480 0x555d27c8b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c85bf0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa654ff9700 0 client.26798 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:15:29.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3262273186 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c7b380 0x555d27c86130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 >> 172.21.15.90:0/3262273186 conn(0x555d27bd86a0 msgr2=0x555d27c73c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 shutdown_connections 2024-04-06T14:15:29.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.781+0000 7fa67f518b80 1 -- 172.21.15.90:0/3262273186 wait complete. 2024-04-06T14:15:29.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.782+0000 7fa67f518b80 20 client.26798 trim_cache size 0 max 16384 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 Processor -- start 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 -- start start 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c8b930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c8a880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27c80300 con 0x555d27c8c480 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27c7c3d0 con 0x555d27c79cc0 2024-04-06T14:15:29.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.799+0000 7fa67f518b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27c76210 con 0x555d27c7b380 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54806/0 (socket says 172.21.15.90:54806) 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 -- 172.21.15.90:0/3187056905 learned_addr learned my addr 172.21.15.90:0/3187056905 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa6708ff700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c8a880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671100700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c8b930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 -- 172.21.15.90:0/3187056905 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 msgr2=0x555d27c8a880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c8a880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 -- 172.21.15.90:0/3187056905 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 msgr2=0x555d27c8b930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c8b930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671901700 1 -- 172.21.15.90:0/3187056905 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555d27b90ea0 con 0x555d27c8c480 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.800+0000 7fa671100700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c8b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa671901700 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7fa664007960 tx=0x7fa664009ac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b825cf96142a8d70 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa657fff700 1 -- 172.21.15.90:0/3187056905 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa66400e040 con 0x555d27c8c480 2024-04-06T14:15:29.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa657fff700 1 -- 172.21.15.90:0/3187056905 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa664043070 con 0x555d27c8c480 2024-04-06T14:15:29.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa657fff700 1 -- 172.21.15.90:0/3187056905 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa664005040 con 0x555d27c8c480 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa67f518b80 1 -- 172.21.15.90:0/3187056905 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 msgr2=0x555d27c7f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.801+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 secure :-1 s=READY pgs=719 cs=0 l=1 rev1=1 crypto rx=0x7fa664007960 tx=0x7fa664009ac0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 -- 172.21.15.90:0/3187056905 shutdown_connections 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c7f020 unknown :-1 s=CLOSED pgs=719 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c79cc0 0x555d27c8b930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 --2- 172.21.15.90:0/3187056905 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c7b380 0x555d27c8a880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 -- 172.21.15.90:0/3187056905 >> 172.21.15.90:0/3187056905 conn(0x555d27bd86a0 msgr2=0x555d27c807e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 -- 172.21.15.90:0/3187056905 shutdown_connections 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 -- 172.21.15.90:0/3187056905 wait complete. 2024-04-06T14:15:29.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 -- start start 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.802+0000 7fa67f518b80 1 Processor -- start 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c7b380 0x555d27c89340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c89880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27c8ac40 con 0x555d27c8c480 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27cf84f0 con 0x555d27c7b380 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555d27cf8660 con 0x555d27c79cc0 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa67f518b80 10 client.0 _ll_register_callbacks cb 0x555d27b70470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa671100700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa671100700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58808/0 (socket says 172.21.15.90:58808) 2024-04-06T14:15:29.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.803+0000 7fa671100700 1 -- 172.21.15.90:0/4247627937 learned_addr learned my addr 172.21.15.90:0/4247627937 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.804+0000 7fa671100700 1 -- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c7b380 msgr2=0x555d27c89340 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.804+0000 7fa671100700 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c7b380 0x555d27c89340 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.804+0000 7fa671100700 1 -- 172.21.15.90:0/4247627937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 msgr2=0x555d27c89880 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.804+0000 7fa671100700 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c89880 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.804+0000 7fa671100700 1 -- 172.21.15.90:0/4247627937 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x555d279f8b60 con 0x555d27c79cc0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa671100700 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa66809b010 tx=0x7fa6680038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1f9cb57c542b8d91 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa668003680 con 0x555d27c79cc0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa66809e0a0 con 0x555d27c79cc0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa6680af070 con 0x555d27c79cc0 2024-04-06T14:15:29.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa66809b040 con 0x555d27c79cc0 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 10 client.26590 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fa66c10d370 con 0x555d27c79cc0 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa6680af070 con 0x555d27c79cc0 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa668096980 con 0x555d27c79cc0 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 10 client.26590 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 10 client.26590 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 20 client.26590 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27cf9980 0x555d27cfbd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.805+0000 7fa67f518b80 4 client.26590 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x555d27a19cf0 con 0x555d27cf9980 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa6708ff700 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27cf9980 0x555d27cfbd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa654ff9700 10 client.26590 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:15:29.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa6708ff700 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27cf9980 0x555d27cfbd70 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a7f23b9100832460 server_cookie=6b2d9a6e5136a3fa in_seq=0 out_seq=0 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa654ff9700 1 -- 172.21.15.90:0/4247627937 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x555d27a19cf0 con 0x555d27cf9980 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa654ff9700 10 client.26590 handle_command_reply: tid=0 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa67f518b80 1 client.26590 shutdown 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 msgr2=0x555d27c88e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.806+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa66809b010 tx=0x7fa6680038a0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 shutdown_connections 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555d27cf9980 0x555d27cfbd70 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555d27c8c480 0x555d27c89880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555d27c79cc0 0x555d27c88e00 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 --2- 172.21.15.90:0/4247627937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555d27c7b380 0x555d27c89340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa654ff9700 0 client.26590 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.807+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 >> 172.21.15.90:0/4247627937 conn(0x555d27bd86a0 msgr2=0x555d27c74500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 shutdown_connections 2024-04-06T14:15:29.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa67f518b80 1 -- 172.21.15.90:0/4247627937 wait complete. 2024-04-06T14:15:29.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa67f518b80 20 client.26590 trim_cache size 0 max 16384 2024-04-06T14:15:29.812 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 msgr2=0x7fa6580a59a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 0x7fa6580a59a0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fa66c196660 tx=0x7fa65c009310 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 msgr2=0x7fa66c195110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fa66800eb30 tx=0x7fa66800ee40 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 shutdown_connections 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa6580a34e0 0x7fa6580a59a0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa66c1999f0 0x7fa66c199e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa66c0ffe60 0x7fa66c195110 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 --2- 172.21.15.90:0/3385430647 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa66c104f80 0x7fa66c195650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 >> 172.21.15.90:0/3385430647 conn(0x7fa66c076db0 msgr2=0x7fa66c0fdcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.808+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 shutdown_connections 2024-04-06T14:15:29.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:29.809+0000 7fa673364700 1 -- 172.21.15.90:0/3385430647 wait complete. 2024-04-06T14:15:29.826 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:15:29.826 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:15:29.857 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:29 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2621428250' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.004+0000 7fb04c148700 1 -- 172.21.15.90:0/4260841242 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 msgr2=0x7fb04410caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.004+0000 7fb04c148700 1 --2- 172.21.15.90:0/4260841242 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04410caf0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fb03c00f400 tx=0x7fb03c00f710 comp rx=0 tx=0).stop 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 -- 172.21.15.90:0/4260841242 shutdown_connections 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 --2- 172.21.15.90:0/4260841242 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04410caf0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 --2- 172.21.15.90:0/4260841242 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0440ffe70 0x7fb044100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 --2- 172.21.15.90:0/4260841242 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb044100820 0x7fb044104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 -- 172.21.15.90:0/4260841242 >> 172.21.15.90:0/4260841242 conn(0x7fb044076d50 msgr2=0x7fb044077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 -- 172.21.15.90:0/4260841242 shutdown_connections 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 -- 172.21.15.90:0/4260841242 wait complete. 2024-04-06T14:15:30.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.005+0000 7fb04c148700 1 Processor -- start 2024-04-06T14:15:30.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 -- start start 2024-04-06T14:15:30.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04419dcd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0441a19b0 con 0x7fb0440ffe70 2024-04-06T14:15:30.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0441a1b20 con 0x7fb044104f70 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04c148700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0441a1ce0 con 0x7fb0441a0fc0 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54826/0 (socket says 172.21.15.90:54826) 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58830/0 (socket says 172.21.15.90:58830) 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb049ee4700 1 -- 172.21.15.90:0/3914444600 learned_addr learned my addr 172.21.15.90:0/3914444600 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.006+0000 7fb0496e3700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04419dcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 -- 172.21.15.90:0/3914444600 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 msgr2=0x7fb0441a1470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 -- 172.21.15.90:0/3914444600 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 msgr2=0x7fb04419dcd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04419dcd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 -- 172.21.15.90:0/3914444600 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb03c00f0b0 con 0x7fb0440ffe70 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb0496e3700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04419dcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:30.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb04a6e5700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fb04000c920 tx=0x7fb04000cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb03affd700 1 -- 172.21.15.90:0/3914444600 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb040004a30 con 0x7fb0440ffe70 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0441aa350 con 0x7fb0440ffe70 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb03affd700 1 -- 172.21.15.90:0/3914444600 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb0400054e0 con 0x7fb0440ffe70 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb03affd700 1 -- 172.21.15.90:0/3914444600 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb040007bb0 con 0x7fb0440ffe70 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.007+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0441aa860 con 0x7fb0440ffe70 2024-04-06T14:15:30.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.010+0000 7fb03affd700 1 -- 172.21.15.90:0/3914444600 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb040007d10 con 0x7fb0440ffe70 2024-04-06T14:15:30.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.011+0000 7fb03affd700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 0x7fb0300a3780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.011+0000 7fb03affd700 1 -- 172.21.15.90:0/3914444600 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb0400cc7b0 con 0x7fb0440ffe70 2024-04-06T14:15:30.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.011+0000 7fb0582fcb80 1 Processor -- start 2024-04-06T14:15:30.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.012+0000 7fb0496e3700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 0x7fb0300a3780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.012+0000 7fb0496e3700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 0x7fb0300a3780 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fb04419ece0 tx=0x7fb03400d040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.012+0000 7fb0582fcb80 1 -- start start 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 0x561569817b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 0x5615698218a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 0x5615698241d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56156976f990 con 0x5615698180a0 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56156976d350 con 0x561569821de0 2024-04-06T14:15:30.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561569768c40 con 0x561569817780 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 0x561569817b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb0496e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 0x5615698218a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 0x561569817b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58832/0 (socket says 172.21.15.90:58832) 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb049ee4700 1 -- 172.21.15.90:0/2434467518 learned_addr learned my addr 172.21.15.90:0/2434467518 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.013+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 0x5615698241d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 -- 172.21.15.90:0/2434467518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 msgr2=0x561569817b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 0x561569817b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 -- 172.21.15.90:0/2434467518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 msgr2=0x5615698218a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 0x5615698218a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 -- 172.21.15.90:0/2434467518 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5615695ea3b0 con 0x561569821de0 2024-04-06T14:15:30.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb0496e3700 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 0x5615698218a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:30.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb04a6e5700 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 0x5615698241d0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fb03c0060b0 tx=0x7fb03c006000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=16c703ed6c8a042b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb02ffff700 1 -- 172.21.15.90:0/2434467518 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb03c012070 con 0x561569821de0 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb02ffff700 1 -- 172.21.15.90:0/2434467518 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb03c00d5d0 con 0x561569821de0 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.014+0000 7fb02ffff700 1 -- 172.21.15.90:0/2434467518 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb03c008450 con 0x561569821de0 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2434467518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 msgr2=0x5615698241d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 0x5615698241d0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fb03c0060b0 tx=0x7fb03c006000 comp rx=0 tx=0).stop 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2434467518 shutdown_connections 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569821de0 0x5615698241d0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561569817780 0x561569817b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2434467518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5615698180a0 0x5615698218a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2434467518 >> 172.21.15.90:0/2434467518 conn(0x5615697766a0 msgr2=0x561569812a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2434467518 shutdown_connections 2024-04-06T14:15:30.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.015+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2434467518 wait complete. 2024-04-06T14:15:30.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.016+0000 7fb0582fcb80 1 -- start start 2024-04-06T14:15:30.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.016+0000 7fb0582fcb80 1 Processor -- start 2024-04-06T14:15:30.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x56156981ebd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981f110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981f650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561569704d80 con 0x561569821de0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561569704a20 con 0x561569817780 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5615695ec6a0 con 0x5615698180a0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0582fcb80 10 client.0 _ll_register_callbacks cb 0x56156970e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x56156981ebd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb049ee4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x56156981ebd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43610/0 (socket says 172.21.15.90:43610) 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb049ee4700 1 -- 172.21.15.90:0/2099493875 learned_addr learned my addr 172.21.15.90:0/2099493875 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb04a6e5700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981f650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 -- 172.21.15.90:0/2099493875 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 msgr2=0x56156981ebd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x56156981ebd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 -- 172.21.15.90:0/2099493875 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 msgr2=0x56156981f650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981f650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.017+0000 7fb0496e3700 1 -- 172.21.15.90:0/2099493875 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb034011bd0 con 0x5615698180a0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb0496e3700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981f110 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fb04406a4e0 tx=0x7fb034004750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ed43890124a4e2cc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb02d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb034010040 con 0x5615698180a0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb03401c040 con 0x5615698180a0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x561569729380 con 0x5615698180a0 2024-04-06T14:15:30.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.018+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb034015be0 con 0x5615698180a0 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 10 client.26599 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fb04410d360 con 0x5615698180a0 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x561569729380 con 0x5615698180a0 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb03402c410 con 0x5615698180a0 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 10 client.26599 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 10 client.26599 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 20 client.26599 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x561569831b40 0x561569833f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 4 client.26599 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.019+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561569761700 con 0x561569831b40 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.020+0000 7fb049ee4700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x561569831b40 0x561569833f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.020+0000 7fb02d7fa700 10 client.26599 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.020+0000 7fb049ee4700 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x561569831b40 0x561569833f60 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=1b6cd6889f41d1e server_cookie=6c6a750150a74e63 in_seq=0 out_seq=0 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb02d7fa700 1 -- 172.21.15.90:0/2099493875 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x561569761700 con 0x561569831b40 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb02d7fa700 10 client.26599 handle_command_reply: tid=0 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 client.26599 shutdown 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 msgr2=0x56156981f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981f110 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fb04406a4e0 tx=0x7fb034004750 comp rx=0 tx=0).stop 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 shutdown_connections 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x561569831b40 0x561569833f60 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x56156981ebd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/2099493875 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981f110 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 >> 172.21.15.90:0/2099493875 conn(0x5615697766a0 msgr2=0x561569811c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb02d7fa700 0 client.26599 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 shutdown_connections 2024-04-06T14:15:30.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.021+0000 7fb0582fcb80 1 -- 172.21.15.90:0/2099493875 wait complete. 2024-04-06T14:15:30.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.022+0000 7fb0582fcb80 20 client.26599 trim_cache size 0 max 16384 2024-04-06T14:15:30.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.039+0000 7fb0582fcb80 1 Processor -- start 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.039+0000 7fb0582fcb80 1 -- start start 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569813dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981e130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981d4c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56156981d100 con 0x561569821de0 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56156981d270 con 0x561569817780 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56156980ec50 con 0x5615698180a0 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981d4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981d4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54868/0 (socket says 172.21.15.90:54868) 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb04a6e5700 1 -- 172.21.15.90:0/3433950584 learned_addr learned my addr 172.21.15.90:0/3433950584 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569813dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.040+0000 7fb0496e3700 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981e130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 -- 172.21.15.90:0/3433950584 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 msgr2=0x56156981e130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981e130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 -- 172.21.15.90:0/3433950584 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 msgr2=0x56156981d4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981d4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 -- 172.21.15.90:0/3433950584 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561569596b60 con 0x561569817780 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb049ee4700 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569813dc0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fb0400c9f90 tx=0x7fb040095d10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e91f875e85684b5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb038ff9700 1 -- 172.21.15.90:0/3433950584 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0400b2260 con 0x561569817780 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb038ff9700 1 -- 172.21.15.90:0/3433950584 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb0400b7e90 con 0x561569817780 2024-04-06T14:15:30.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.041+0000 7fb038ff9700 1 -- 172.21.15.90:0/3433950584 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb040095fb0 con 0x561569817780 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- 172.21.15.90:0/3433950584 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 msgr2=0x561569813dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569813dc0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fb0400c9f90 tx=0x7fb040095d10 comp rx=0 tx=0).stop 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- 172.21.15.90:0/3433950584 shutdown_connections 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x56156981d4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569813dc0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/3433950584 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x56156981e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- 172.21.15.90:0/3433950584 >> 172.21.15.90:0/3433950584 conn(0x5615697766a0 msgr2=0x56156982d150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- 172.21.15.90:0/3433950584 shutdown_connections 2024-04-06T14:15:30.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- 172.21.15.90:0/3433950584 wait complete. 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 -- start start 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.042+0000 7fb0582fcb80 1 Processor -- start 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569828720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x561569826b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x5615698270a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5615698275e0 con 0x561569821de0 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561569827750 con 0x561569817780 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561569827890 con 0x5615698180a0 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0582fcb80 10 client.0 _ll_register_callbacks cb 0x56156970e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0496e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x561569826b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0496e3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x561569826b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58852/0 (socket says 172.21.15.90:58852) 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb04a6e5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x5615698270a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.043+0000 7fb0496e3700 1 -- 172.21.15.90:0/4110137396 learned_addr learned my addr 172.21.15.90:0/4110137396 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569828720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 -- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 msgr2=0x561569826b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x561569826b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 -- 172.21.15.90:0/4110137396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 msgr2=0x5615698270a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x5615698270a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.044+0000 7fb049ee4700 1 -- 172.21.15.90:0/4110137396 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0441aa860 con 0x561569817780 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb049ee4700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569828720 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fb0400995a0 tx=0x7fb04009a080 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7b96bba95b9bc283 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb02d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb0400a1d20 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb040097c40 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb03c00d5d0 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.045+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb040005650 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 10 client.26819 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56156972eea0 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb03c00d5d0 con 0x561569817780 2024-04-06T14:15:30.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb040099b00 con 0x561569817780 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 10 client.26819 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 10 client.26819 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 20 client.26819 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5615698978a0 0x561569899c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 4 client.26819 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5615695b7cf0 con 0x5615698978a0 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb0496e3700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5615698978a0 0x561569899c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.046+0000 7fb02d7fa700 10 client.26819 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0496e3700 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5615698978a0 0x561569899c90 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2bb59c5c1182f6ba server_cookie=64a34abea674c60a in_seq=0 out_seq=0 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb02d7fa700 1 -- 172.21.15.90:0/4110137396 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5615695b7cf0 con 0x5615698978a0 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb02d7fa700 10 client.26819 handle_command_reply: tid=0 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 client.26819 shutdown 2024-04-06T14:15:30.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 msgr2=0x561569828720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569828720 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fb0400995a0 tx=0x7fb04009a080 comp rx=0 tx=0).stop 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 shutdown_connections 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x5615698978a0 0x561569899c90 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561569821de0 0x5615698270a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb02d7fa700 0 client.26819 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561569817780 0x561569828720 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 --2- 172.21.15.90:0/4110137396 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5615698180a0 0x561569826b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 >> 172.21.15.90:0/4110137396 conn(0x5615697766a0 msgr2=0x56156981d460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 shutdown_connections 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 1 -- 172.21.15.90:0/4110137396 wait complete. 2024-04-06T14:15:30.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.047+0000 7fb0582fcb80 20 client.26819 trim_cache size 0 max 16384 2024-04-06T14:15:30.052 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:15:30.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.050+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 msgr2=0x7fb0300a3780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.050+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 0x7fb0300a3780 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7fb04419ece0 tx=0x7fb03400d040 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.050+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 msgr2=0x7fb04419d790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.050+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 secure :-1 s=READY pgs=720 cs=0 l=1 rev1=1 crypto rx=0x7fb04000c920 tx=0x7fb04000cc30 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 shutdown_connections 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb0300a12c0 0x7fb0300a3780 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb0441a0fc0 0x7fb0441a1470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb0440ffe70 0x7fb04419d790 unknown :-1 s=CLOSED pgs=720 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 --2- 172.21.15.90:0/3914444600 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb044104f70 0x7fb04419dcd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 >> 172.21.15.90:0/3914444600 conn(0x7fb044076d50 msgr2=0x7fb0440fdf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 shutdown_connections 2024-04-06T14:15:30.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.051+0000 7fb04c148700 1 -- 172.21.15.90:0/3914444600 wait complete. 2024-04-06T14:15:30.065 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:15:30.065 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:15:30.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:29 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2621428250' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:15:30.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:29 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2621428250' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:15:30.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.290+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2057283085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 msgr2=0x7fcad41060e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.290+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2057283085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad41060e0 secure :-1 s=READY pgs=721 cs=0 l=1 rev1=1 crypto rx=0x7fcad0009990 tx=0x7fcad0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2057283085 shutdown_connections 2024-04-06T14:15:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2057283085 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ffbe0 0x7fcad410dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2057283085 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad41060e0 unknown :-1 s=CLOSED pgs=721 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2057283085 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ff220 0x7fcad40ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2057283085 >> 172.21.15.90:0/2057283085 conn(0x7fcad40761b0 msgr2=0x7fcad40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2057283085 shutdown_connections 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.291+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2057283085 wait complete. 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 Processor -- start 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 -- start start 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 0x7fcad4100400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ffbe0 0x7fcad4100940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcad41a55a0 con 0x7fcad4105d00 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcad41a5710 con 0x7fcad40ff220 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcadc2f9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcad41a58d0 con 0x7fcad40ffbe0 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcada896700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.294 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcada095700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 0x7fcad4100400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcada896700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54884/0 (socket says 172.21.15.90:54884) 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcad9894700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ffbe0 0x7fcad4100940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.292+0000 7fcada896700 1 -- 172.21.15.90:0/2048134829 learned_addr learned my addr 172.21.15.90:0/2048134829 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 -- 172.21.15.90:0/2048134829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ffbe0 msgr2=0x7fcad4100940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ffbe0 0x7fcad4100940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 -- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 msgr2=0x7fcad4100e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 -- 172.21.15.90:0/2048134829 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcad00095f0 con 0x7fcad40ff220 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada896700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcada095700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 0x7fcad4100400 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fcad00059d0 tx=0x7fcad0004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcacb7fe700 1 -- 172.21.15.90:0/2048134829 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad001d070 con 0x7fcad40ff220 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcacb7fe700 1 -- 172.21.15.90:0/2048134829 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcad0022470 con 0x7fcad40ff220 2024-04-06T14:15:30.295 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcad41a5b50 con 0x7fcad40ff220 2024-04-06T14:15:30.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcacb7fe700 1 -- 172.21.15.90:0/2048134829 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad0017650 con 0x7fcad40ff220 2024-04-06T14:15:30.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.293+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcad41a5f70 con 0x7fcad40ff220 2024-04-06T14:15:30.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcacb7fe700 1 -- 172.21.15.90:0/2048134829 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fcad000fb10 con 0x7fcad40ff220 2024-04-06T14:15:30.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 Processor -- start 2024-04-06T14:15:30.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 -- start start 2024-04-06T14:15:30.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110eb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 0x563be11188a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be111b1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcacb7fe700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 0x7fcac00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be1066990 con 0x563be1118de0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be1064350 con 0x563be110f0a0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcae84adb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be105fc40 con 0x563be110e780 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcada896700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 0x7fcac00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcada896700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.297+0000 7fcada095700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 0x563be11188a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada896700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58894/0 (socket says 172.21.15.90:58894) 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada896700 1 -- 172.21.15.90:0/2228399053 learned_addr learned my addr 172.21.15.90:0/2228399053 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcad9894700 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be111b1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcacb7fe700 1 -- 172.21.15.90:0/2048134829 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcad00cde50 con 0x7fcad40ff220 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 -- 172.21.15.90:0/2228399053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 msgr2=0x563be110eb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110eb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 -- 172.21.15.90:0/2228399053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 msgr2=0x563be111b1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be111b1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 -- 172.21.15.90:0/2228399053 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563be0ee13b0 con 0x563be110f0a0 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada896700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 0x7fcac00a3750 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fcacc009550 tx=0x7fcacc0091c0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcada095700 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 0x563be11188a0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7fcad00cb520 tx=0x7fcad006e100 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d9a690de264b2b90 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcac8da5700 1 -- 172.21.15.90:0/2228399053 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad009aab0 con 0x563be110f0a0 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcac8da5700 1 -- 172.21.15.90:0/2228399053 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcad009a8d0 con 0x563be110f0a0 2024-04-06T14:15:30.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.298+0000 7fcac8da5700 1 -- 172.21.15.90:0/2228399053 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcad00a3360 con 0x563be110f0a0 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 -- 172.21.15.90:0/2228399053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 msgr2=0x563be11188a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 0x563be11188a0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7fcad00cb520 tx=0x7fcad006e100 comp rx=0 tx=0).stop 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 -- 172.21.15.90:0/2228399053 shutdown_connections 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be111b1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110eb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2228399053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110f0a0 0x563be11188a0 secure :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7fcad00cb520 tx=0x7fcad006e100 comp rx=0 tx=0).stop 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.299+0000 7fcae84adb80 1 -- 172.21.15.90:0/2228399053 >> 172.21.15.90:0/2228399053 conn(0x563be106d6a0 msgr2=0x563be1109a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.300+0000 7fcae84adb80 1 -- 172.21.15.90:0/2228399053 shutdown_connections 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.300+0000 7fcae84adb80 1 -- 172.21.15.90:0/2228399053 wait complete. 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 -- start start 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 Processor -- start 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 0x563be1115cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be1116730 0x563be1124800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be0ffbd80 con 0x563be1118de0 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be0ffba20 con 0x563be110e780 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be0ee36a0 con 0x563be1116730 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcae84adb80 10 client.0 _ll_register_callbacks cb 0x563be1005470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:30.306 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54912/0 (socket says 172.21.15.90:54912) 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 -- 172.21.15.90:0/760011774 learned_addr learned my addr 172.21.15.90:0/760011774 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcada095700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 0x563be1115cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcada896700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be1116730 0x563be1124800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 -- 172.21.15.90:0/760011774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be1116730 msgr2=0x563be1124800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be1116730 0x563be1124800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 -- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 msgr2=0x563be1115cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 0x563be1115cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.301+0000 7fcad9894700 1 -- 172.21.15.90:0/760011774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcad41a5f70 con 0x563be1118de0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcada095700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 0x563be1115cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcad9894700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7fcac4012840 tx=0x7fcac4012b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7387c52e5f2789b1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcac4017070 con 0x563be1118de0 2024-04-06T14:15:30.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563be1020380 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcac4004830 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcac4004a30 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 10 client.17130 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563be0e8db60 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563be1020380 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcac400f4e0 con 0x563be1118de0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 10 client.17130 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 10 client.17130 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 20 client.17130 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1128be0 0x563be112b000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 4 client.17130 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.302+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563be1058700 con 0x563be1128be0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.303+0000 7fcada095700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1128be0 0x563be112b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.303+0000 7fcada095700 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1128be0 0x563be112b000 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6f0ae1d92ab90ddf server_cookie=bd5e4134789f861c in_seq=0 out_seq=0 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.303+0000 7fcabdffb700 10 client.17130 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcabdffb700 1 -- 172.21.15.90:0/760011774 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563be1058700 con 0x563be1128be0 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcabdffb700 10 client.17130 handle_command_reply: tid=0 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 client.17130 shutdown 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 msgr2=0x563be11161f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 secure :-1 s=READY pgs=722 cs=0 l=1 rev1=1 crypto rx=0x7fcac4012840 tx=0x7fcac4012b50 comp rx=0 tx=0).stop 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 shutdown_connections 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1128be0 0x563be112b000 crc :-1 s=READY pgs=179 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be1116730 0x563be1124800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcabdffb700 0 client.17130 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be110e780 0x563be1115cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 --2- 172.21.15.90:0/760011774 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11161f0 unknown :-1 s=CLOSED pgs=722 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 >> 172.21.15.90:0/760011774 conn(0x563be106d6a0 msgr2=0x563be1108c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 shutdown_connections 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.304+0000 7fcae84adb80 1 -- 172.21.15.90:0/760011774 wait complete. 2024-04-06T14:15:30.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.305+0000 7fcae84adb80 20 client.17130 trim_cache size 0 max 16384 2024-04-06T14:15:30.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.319+0000 7fcae84adb80 1 Processor -- start 2024-04-06T14:15:30.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.319+0000 7fcae84adb80 1 -- start start 2024-04-06T14:15:30.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110de80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11098e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be110b1b0 con 0x563be1118de0 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be1116550 con 0x563be1116730 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcae84adb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be11117f0 con 0x563be110e780 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43670/0 (socket says 172.21.15.90:43670) 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 -- 172.21.15.90:0/2229299503 learned_addr learned my addr 172.21.15.90:0/2229299503 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcada896700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11098e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcada095700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 -- 172.21.15.90:0/2229299503 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 msgr2=0x563be110de80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110de80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 -- 172.21.15.90:0/2229299503 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 msgr2=0x563be11098e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11098e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcad9894700 1 -- 172.21.15.90:0/2229299503 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563be1025ea0 con 0x563be1116730 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.320+0000 7fcada095700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcad9894700 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fcac400f940 tx=0x7fcac400fdf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4fe7520477d75b5c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcac95a6700 1 -- 172.21.15.90:0/2229299503 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcac4017070 con 0x563be1116730 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcac95a6700 1 -- 172.21.15.90:0/2229299503 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcac4018040 con 0x563be1116730 2024-04-06T14:15:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcac95a6700 1 -- 172.21.15.90:0/2229299503 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcac40038e0 con 0x563be1116730 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 -- 172.21.15.90:0/2229299503 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 msgr2=0x563be11093a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7fcac400f940 tx=0x7fcac400fdf0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 -- 172.21.15.90:0/2229299503 shutdown_connections 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1118de0 0x563be11098e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be110de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2229299503 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1116730 0x563be11093a0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.321+0000 7fcae84adb80 1 -- 172.21.15.90:0/2229299503 >> 172.21.15.90:0/2229299503 conn(0x563be106d6a0 msgr2=0x563be110d770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- 172.21.15.90:0/2229299503 shutdown_connections 2024-04-06T14:15:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- 172.21.15.90:0/2229299503 wait complete. 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- start start 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 Processor -- start 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be111d6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1118de0 0x563be117e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be117e730 con 0x563be1116730 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be117e870 con 0x563be1118de0 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563be117ea30 con 0x563be110e780 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcae84adb80 10 client.0 _ll_register_callbacks cb 0x563be1005470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54924/0 (socket says 172.21.15.90:54924) 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 -- 172.21.15.90:0/2965735967 learned_addr learned my addr 172.21.15.90:0/2965735967 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcada896700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1118de0 0x563be117e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 -- 172.21.15.90:0/2965735967 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 msgr2=0x563be111d6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be111d6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 -- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1118de0 msgr2=0x563be117e1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.322+0000 7fcad9894700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1118de0 0x563be117e1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcad9894700 1 -- 172.21.15.90:0/2965735967 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x563be0e8db60 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcad9894700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fcac4012840 tx=0x7fcac4009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c377f6f8e414353 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcac4017040 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcac4005440 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fcad009a8d0 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcac4005600 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 10 client.17136 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563be0ee5cd0 con 0x563be1116730 2024-04-06T14:15:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcac403d080 con 0x563be1116730 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcac401c810 con 0x563be1116730 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 10 client.17136 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 10 client.17136 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 20 client.17136 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1185ef0 0x563be1188310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 4 client.17136 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.323+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x563be0eaecf0 con 0x563be1185ef0 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.324+0000 7fcada095700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1185ef0 0x563be1188310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.324+0000 7fcada095700 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1185ef0 0x563be1188310 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=47c8edf411d9f4c3 server_cookie=37312c479de01f14 in_seq=0 out_seq=0 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcabdffb700 10 client.17136 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcabdffb700 1 -- 172.21.15.90:0/2965735967 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x563be0eaecf0 con 0x563be1185ef0 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcabdffb700 10 client.17136 handle_command_reply: tid=0 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 client.17136 shutdown 2024-04-06T14:15:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 msgr2=0x563be111dc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 secure :-1 s=READY pgs=723 cs=0 l=1 rev1=1 crypto rx=0x7fcac4012840 tx=0x7fcac4009310 comp rx=0 tx=0).stop 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 shutdown_connections 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563be1118de0 0x563be117e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563be1185ef0 0x563be1188310 crc :-1 s=READY pgs=180 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563be110e780 0x563be111d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 --2- 172.21.15.90:0/2965735967 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563be1116730 0x563be111dc20 unknown :-1 s=CLOSED pgs=723 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcabdffb700 0 client.17136 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 >> 172.21.15.90:0/2965735967 conn(0x563be106d6a0 msgr2=0x563be1115510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 shutdown_connections 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 1 -- 172.21.15.90:0/2965735967 wait complete. 2024-04-06T14:15:30.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.325+0000 7fcae84adb80 20 client.17136 trim_cache size 0 max 16384 2024-04-06T14:15:30.333 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:15:30.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 msgr2=0x7fcac00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 0x7fcac00a3750 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7fcacc009550 tx=0x7fcacc0091c0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 msgr2=0x7fcad4100400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 0x7fcad4100400 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7fcad00059d0 tx=0x7fcad0004730 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 shutdown_connections 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcac00a1290 0x7fcac00a3750 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcad4105d00 0x7fcad4100e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcad40ff220 0x7fcad4100400 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 --2- 172.21.15.90:0/2048134829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcad40ffbe0 0x7fcad4100940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 >> 172.21.15.90:0/2048134829 conn(0x7fcad40761b0 msgr2=0x7fcad4103e20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 shutdown_connections 2024-04-06T14:15:30.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:30.326+0000 7fcadc2f9700 1 -- 172.21.15.90:0/2048134829 wait complete. 2024-04-06T14:15:31.145 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:30 smithi134 ceph-mon[47549]: pgmap v314: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 15 MiB/s wr, 862 op/s 2024-04-06T14:15:31.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:30 smithi105 ceph-mon[46969]: pgmap v314: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 15 MiB/s wr, 862 op/s 2024-04-06T14:15:31.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:30 smithi090 ceph-mon[43420]: pgmap v314: 97 pgs: 97 active+clean; 6.6 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 15 MiB/s wr, 862 op/s 2024-04-06T14:15:31.340 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:15:31.340 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:15:31.341 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 -- 172.21.15.90:0/275703803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107d50 msgr2=0x7fb3cc1141f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/275703803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107d50 0x7fb3cc1141f0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400b0a0 tx=0x7fb3c400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 -- 172.21.15.90:0/275703803 shutdown_connections 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/275703803 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107d50 0x7fb3cc1141f0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/275703803 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc10d450 0x7fb3cc10d830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/275703803 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc107390 0x7fb3cc107810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.537+0000 7fb3d21b1700 1 -- 172.21.15.90:0/275703803 >> 172.21.15.90:0/275703803 conn(0x7fb3cc076ba0 msgr2=0x7fb3cc105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.538+0000 7fb3d21b1700 1 -- 172.21.15.90:0/275703803 shutdown_connections 2024-04-06T14:15:31.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.538+0000 7fb3d21b1700 1 -- 172.21.15.90:0/275703803 wait complete. 2024-04-06T14:15:31.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.538+0000 7fb3d21b1700 1 Processor -- start 2024-04-06T14:15:31.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 -- start start 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107390 0x7fb3cc0724f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 0x7fb3cc072a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 0x7fb3cc06b7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3cc06bf90 con 0x7fb3cc107d50 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3cc06c100 con 0x7fb3cc10d450 2024-04-06T14:15:31.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.539+0000 7fb3d21b1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3cc06c2c0 con 0x7fb3cc107390 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3caffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 0x7fb3cc072a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3caffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 0x7fb3cc072a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54940/0 (socket says 172.21.15.90:54940) 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3caffd700 1 -- 172.21.15.90:0/2401282625 learned_addr learned my addr 172.21.15.90:0/2401282625 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 0x7fb3cc06b7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 -- 172.21.15.90:0/2401282625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107390 msgr2=0x7fb3cc0724f0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107390 0x7fb3cc0724f0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 -- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 msgr2=0x7fb3cc072a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 0x7fb3cc072a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 -- 172.21.15.90:0/2401282625 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3c4009d00 con 0x7fb3cc10d450 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.540+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 0x7fb3cc06b7c0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400b0a0 tx=0x7fb3c4007910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.541+0000 7fb3c8ff9700 1 -- 172.21.15.90:0/2401282625 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3c400e040 con 0x7fb3cc10d450 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.541+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb3cc070080 con 0x7fb3cc10d450 2024-04-06T14:15:31.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.541+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3cc070530 con 0x7fb3cc10d450 2024-04-06T14:15:31.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.544+0000 7fb3c8ff9700 1 -- 172.21.15.90:0/2401282625 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3c4009b80 con 0x7fb3cc10d450 2024-04-06T14:15:31.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.544+0000 7fb3c8ff9700 1 -- 172.21.15.90:0/2401282625 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3c401ddf0 con 0x7fb3cc10d450 2024-04-06T14:15:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 Processor -- start 2024-04-06T14:15:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3c8ff9700 1 -- 172.21.15.90:0/2401282625 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb3c4004a50 con 0x7fb3cc10d450 2024-04-06T14:15:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 -- start start 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6c120 0x55cd2df75920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df78250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2dec3a10 con 0x55cd2df6c120 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2dec13d0 con 0x55cd2df6b800 2024-04-06T14:15:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.547+0000 7fb3de365b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2debccc0 con 0x55cd2df75e60 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43708/0 (socket says 172.21.15.90:43708) 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 -- 172.21.15.90:0/2051098420 learned_addr learned my addr 172.21.15.90:0/2051098420 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df78250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3c8ff9700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 0x7fb3b40a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 -- 172.21.15.90:0/2051098420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 msgr2=0x55cd2df78250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df78250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 -- 172.21.15.90:0/2051098420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6c120 msgr2=0x55cd2df75920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6c120 0x55cd2df75920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 -- 172.21.15.90:0/2051098420 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cd2dd3e430 con 0x55cd2df6b800 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3c8ff9700 1 -- 172.21.15.90:0/2401282625 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb3c40cde00 con 0x7fb3cc10d450 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7fb3c00046e0 tx=0x7fb3c00047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=307350ee83c02570 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.548+0000 7fb3b1ffb700 1 -- 172.21.15.90:0/2051098420 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3c0004830 con 0x55cd2df6b800 2024-04-06T14:15:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 0x7fb3b40a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3b1ffb700 1 -- 172.21.15.90:0/2051098420 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3c0004e50 con 0x55cd2df6b800 2024-04-06T14:15:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3b1ffb700 1 -- 172.21.15.90:0/2051098420 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3c000e610 con 0x55cd2df6b800 2024-04-06T14:15:31.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 -- 172.21.15.90:0/2051098420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 msgr2=0x55cd2df6bbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3cbfff700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 0x7fb3b40a3890 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400bfd0 tx=0x7fb3c40a1040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7fb3c00046e0 tx=0x7fb3c00047c0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 -- 172.21.15.90:0/2051098420 shutdown_connections 2024-04-06T14:15:31.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df78250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6b800 0x55cd2df6bbe0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2051098420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6c120 0x55cd2df75920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.549+0000 7fb3de365b80 1 -- 172.21.15.90:0/2051098420 >> 172.21.15.90:0/2051098420 conn(0x55cd2deca720 msgr2=0x55cd2df66b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.550+0000 7fb3de365b80 1 -- 172.21.15.90:0/2051098420 shutdown_connections 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.550+0000 7fb3de365b80 1 -- 172.21.15.90:0/2051098420 wait complete. 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.550+0000 7fb3de365b80 1 -- start start 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.550+0000 7fb3de365b80 1 Processor -- start 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2df75860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 0x55cd2df72c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 0x55cd2df732a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2de58e00 con 0x55cd2df6b800 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2de58aa0 con 0x55cd2df75e60 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2dd40720 con 0x55cd2df6c120 2024-04-06T14:15:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3de365b80 10 client.0 _ll_register_callbacks cb 0x55cd2de624f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 0x55cd2df72c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 0x55cd2df732a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 0x55cd2df72c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:58962/0 (socket says 172.21.15.90:58962) 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/4169920114 learned_addr learned my addr 172.21.15.90:0/4169920114 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 -- 172.21.15.90:0/4169920114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 msgr2=0x55cd2df72c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 0x55cd2df72c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 -- 172.21.15.90:0/4169920114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 msgr2=0x55cd2df75860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2df75860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 -- 172.21.15.90:0/4169920114 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3c0019040 con 0x55cd2df75e60 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3caffd700 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 0x55cd2df732a0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fb3c0009f70 tx=0x7fb3c0005a00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1c6465981502c65c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.551+0000 7fb3aaffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3c0025070 con 0x55cd2df75e60 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb3c0004830 con 0x55cd2df75e60 2024-04-06T14:15:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cd2de7d400 con 0x55cd2df75e60 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb3c0005330 con 0x55cd2df75e60 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3de365b80 10 client.26849 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cd2dceabe0 con 0x55cd2df75e60 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3c0031030 con 0x55cd2df75e60 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.552+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb3c002ac50 con 0x55cd2df75e60 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 10 client.26849 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 10 client.26849 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 20 client.26849 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2df859d0 0x55cd2df87df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 4 client.26849 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.553+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cd2deb5780 con 0x55cd2df859d0 2024-04-06T14:15:31.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.556+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2df859d0 0x55cd2df87df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.557+0000 7fb3aaffd700 10 client.26849 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.557+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2df859d0 0x55cd2df87df0 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=dd4a186c61ee0857 server_cookie=d3f6d3799393b788 in_seq=0 out_seq=0 2024-04-06T14:15:31.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.557+0000 7fb3aaffd700 1 -- 172.21.15.90:0/4169920114 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cd2deb5780 con 0x55cd2df859d0 2024-04-06T14:15:31.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.557+0000 7fb3aaffd700 10 client.26849 handle_command_reply: tid=0 2024-04-06T14:15:31.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.557+0000 7fb3de365b80 1 client.26849 shutdown 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 msgr2=0x55cd2df732a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 0x55cd2df732a0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fb3c0009f70 tx=0x7fb3c0005a00 comp rx=0 tx=0).stop 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 shutdown_connections 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df75e60 0x55cd2df732a0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2df75860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2df859d0 0x55cd2df87df0 crc :-1 s=READY pgs=181 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 --2- 172.21.15.90:0/4169920114 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6c120 0x55cd2df72c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 >> 172.21.15.90:0/4169920114 conn(0x55cd2deca720 msgr2=0x55cd2df65ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3aaffd700 0 client.26849 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 shutdown_connections 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 1 -- 172.21.15.90:0/4169920114 wait complete. 2024-04-06T14:15:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.558+0000 7fb3de365b80 20 client.26849 trim_cache size 0 max 16384 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 Processor -- start 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 -- start start 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 0x55cd2df66cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df75e60 0x55cd2df75390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df7d0b0 con 0x55cd2df75e60 2024-04-06T14:15:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df73b80 con 0x55cd2df6c120 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.591+0000 7fb3de365b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df72460 con 0x55cd2df6b800 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43726/0 (socket says 172.21.15.90:43726) 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/2002399337 learned_addr learned my addr 172.21.15.90:0/2002399337 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df75e60 0x55cd2df75390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 0x55cd2df66cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/2002399337 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 msgr2=0x55cd2df66cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 0x55cd2df66cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/2002399337 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df75e60 msgr2=0x55cd2df75390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df75e60 0x55cd2df75390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/2002399337 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cd2df8c9e0 con 0x55cd2df6c120 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3caffd700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 0x55cd2df66cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:15:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fb3bc005dd0 tx=0x7fb3bc00b510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a7d0ceb55c2518a5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3b27fc700 1 -- 172.21.15.90:0/2002399337 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3bc016040 con 0x55cd2df6c120 2024-04-06T14:15:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.592+0000 7fb3b27fc700 1 -- 172.21.15.90:0/2002399337 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3bc01d020 con 0x55cd2df6c120 2024-04-06T14:15:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3b27fc700 1 -- 172.21.15.90:0/2002399337 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3bc0047c0 con 0x55cd2df6c120 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 -- 172.21.15.90:0/2002399337 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 msgr2=0x55cd2df7cb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fb3bc005dd0 tx=0x7fb3bc00b510 comp rx=0 tx=0).stop 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 -- 172.21.15.90:0/2002399337 shutdown_connections 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df75e60 0x55cd2df75390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df6b800 0x55cd2df66cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 --2- 172.21.15.90:0/2002399337 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2df7cb70 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 -- 172.21.15.90:0/2002399337 >> 172.21.15.90:0/2002399337 conn(0x55cd2deca720 msgr2=0x55cd2df665a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 -- 172.21.15.90:0/2002399337 shutdown_connections 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.593+0000 7fb3de365b80 1 -- 172.21.15.90:0/2002399337 wait complete. 2024-04-06T14:15:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.594+0000 7fb3de365b80 1 -- start start 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.594+0000 7fb3de365b80 1 Processor -- start 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.594+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2dfd3fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df7bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df7c3b0 con 0x55cd2df6b800 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df7c520 con 0x55cd2df6c120 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cd2df7c6e0 con 0x55cd2df75e60 2024-04-06T14:15:31.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3de365b80 10 client.0 _ll_register_callbacks cb 0x55cd2de624f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43736/0 (socket says 172.21.15.90:43736) 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/797033255 learned_addr learned my addr 172.21.15.90:0/797033255 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/797033255 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 msgr2=0x55cd2df7bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df7bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/797033255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 msgr2=0x55cd2dfd3fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2dfd3fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.595+0000 7fb3cb7fe700 1 -- 172.21.15.90:0/797033255 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3bc00b870 con 0x55cd2df6c120 2024-04-06T14:15:31.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3cb7fe700 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fb3bc005920 tx=0x7fb3bc0050f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5bdb301bb6fba0ea server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3aaffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb3bc019590 con 0x55cd2df6c120 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb3bc01e030 con 0x55cd2df6c120 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb3c0004e50 con 0x55cd2df6c120 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.596+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb3bc004020 con 0x55cd2df6c120 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.597+0000 7fb3de365b80 10 client.26855 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:31.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.597+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cd2de82f20 con 0x55cd2df6c120 2024-04-06T14:15:31.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.606+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3bc053030 con 0x55cd2df6c120 2024-04-06T14:15:31.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.606+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb3bc019070 con 0x55cd2df6c120 2024-04-06T14:15:31.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 10 client.26855 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:31.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 10 client.26855 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:31.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 20 client.26855 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2dfdbe20 0x55cd2dfde240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 4 client.26855 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.610+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55cd2dd0bd70 con 0x55cd2dfdbe20 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.612+0000 7fb3caffd700 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2dfdbe20 0x55cd2dfde240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.616+0000 7fb3aaffd700 10 client.26855 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.616+0000 7fb3caffd700 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2dfdbe20 0x55cd2dfde240 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2b56ab0e60968d72 server_cookie=1e90bf7ce7d5fa7 in_seq=0 out_seq=0 2024-04-06T14:15:31.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.617+0000 7fb3aaffd700 1 -- 172.21.15.90:0/797033255 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55cd2dd0bd70 con 0x55cd2dfdbe20 2024-04-06T14:15:31.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.617+0000 7fb3aaffd700 10 client.26855 handle_command_reply: tid=0 2024-04-06T14:15:31.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.617+0000 7fb3de365b80 1 client.26855 shutdown 2024-04-06T14:15:31.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 msgr2=0x55cd2dfd4510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fb3bc005920 tx=0x7fb3bc0050f0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 shutdown_connections 2024-04-06T14:15:31.618 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cd2dfdbe20 0x55cd2dfde240 crc :-1 s=READY pgs=182 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cd2df75e60 0x55cd2df7bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cd2df6b800 0x55cd2dfd3fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 --2- 172.21.15.90:0/797033255 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cd2df6c120 0x55cd2dfd4510 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 >> 172.21.15.90:0/797033255 conn(0x55cd2deca720 msgr2=0x55cd2df6b080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.618+0000 7fb3aaffd700 0 client.26855 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.619+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 shutdown_connections 2024-04-06T14:15:31.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.619+0000 7fb3de365b80 1 -- 172.21.15.90:0/797033255 wait complete. 2024-04-06T14:15:31.620 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.620+0000 7fb3de365b80 20 client.26855 trim_cache size 0 max 16384 2024-04-06T14:15:31.621 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:15:31.621 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:15:31.621 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "9bc7d263-857d-4be7-b046-9594139e0583", 2024-04-06T14:15:31.621 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:15:31.621 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:15:31.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 msgr2=0x7fb3b40a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 0x7fb3b40a3890 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400bfd0 tx=0x7fb3c40a1040 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 msgr2=0x7fb3cc06b7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 0x7fb3cc06b7c0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fb3c400b0a0 tx=0x7fb3c4007910 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 shutdown_connections 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb3b40a13d0 0x7fb3b40a3890 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb3cc10d450 0x7fb3cc06b7c0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb3cc107390 0x7fb3cc0724f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 --2- 172.21.15.90:0/2401282625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb3cc107d50 0x7fb3cc072a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 >> 172.21.15.90:0/2401282625 conn(0x7fb3cc076ba0 msgr2=0x7fb3cc077cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 shutdown_connections 2024-04-06T14:15:31.623 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.622+0000 7fb3d21b1700 1 -- 172.21.15.90:0/2401282625 wait complete. 2024-04-06T14:15:31.643 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:15:31.643 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 -- 172.21.15.90:0/1278333119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d04104700 msgr2=0x7f5d0410c280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 --2- 172.21.15.90:0/1278333119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d04104700 0x7f5d0410c280 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f5cfc00d210 tx=0x7f5cfc00d520 comp rx=0 tx=0).stop 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 -- 172.21.15.90:0/1278333119 shutdown_connections 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 --2- 172.21.15.90:0/1278333119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d04104700 0x7f5d0410c280 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 --2- 172.21.15.90:0/1278333119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d040ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.840 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 --2- 172.21.15.90:0/1278333119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d040fff90 0x7f5d04103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 -- 172.21.15.90:0/1278333119 >> 172.21.15.90:0/1278333119 conn(0x7f5d040761b0 msgr2=0x7f5d040fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 -- 172.21.15.90:0/1278333119 shutdown_connections 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 -- 172.21.15.90:0/1278333119 wait complete. 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.838+0000 7f5d0b990700 1 Processor -- start 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 -- start start 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d04104700 0x7f5d041998b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d0419dc50 0x7f5d0419e0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d0419e6d0 con 0x7f5d040ff5e0 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d0419e840 con 0x7f5d0419dc50 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0b990700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5d0419ea00 con 0x7f5d04104700 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54990/0 (socket says 172.21.15.90:54990) 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d04104700 0x7f5d041998b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 -- 172.21.15.90:0/975234296 learned_addr learned my addr 172.21.15.90:0/975234296 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 -- 172.21.15.90:0/975234296 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d04104700 msgr2=0x7f5d041998b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d04104700 0x7f5d041998b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 -- 172.21.15.90:0/975234296 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d0419dc50 msgr2=0x7f5d0419e0d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d0419dc50 0x7f5d0419e0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 -- 172.21.15.90:0/975234296 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cf400a610 con 0x7f5d040ff5e0 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.839+0000 7f5d0972c700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f5d0000e9a0 tx=0x7f5d0000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.840+0000 7f5cfa7fc700 1 -- 172.21.15.90:0/975234296 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d00009790 con 0x7f5d040ff5e0 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.840+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5cfc00be50 con 0x7f5d040ff5e0 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.840+0000 7f5cfa7fc700 1 -- 172.21.15.90:0/975234296 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5d0000cbd0 con 0x7f5d040ff5e0 2024-04-06T14:15:31.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.840+0000 7f5cfa7fc700 1 -- 172.21.15.90:0/975234296 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d00018680 con 0x7f5d040ff5e0 2024-04-06T14:15:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.840+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5d041a6300 con 0x7f5d040ff5e0 2024-04-06T14:15:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.842+0000 7f5cfa7fc700 1 -- 172.21.15.90:0/975234296 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5d0000cd40 con 0x7f5d040ff5e0 2024-04-06T14:15:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.843+0000 7f5cfa7fc700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 0x7f5cf00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.843+0000 7f5cfa7fc700 1 -- 172.21.15.90:0/975234296 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5d00014070 con 0x7f5d040ff5e0 2024-04-06T14:15:31.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.843+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 0x7f5cf00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 Processor -- start 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 -- start start 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 0x7f5cf00a3750 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5d0419a8c0 tx=0x7f5cf4011040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 0x55926e744b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e74e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e7511d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e69c990 con 0x55926e74ede0 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e69a350 con 0x55926e744780 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e695c40 con 0x55926e7450a0 2024-04-06T14:15:31.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d09f2d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e7511d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 0x55926e744b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e74e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d09f2d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e7511d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54994/0 (socket says 172.21.15.90:54994) 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d09f2d700 1 -- 172.21.15.90:0/874678984 learned_addr learned my addr 172.21.15.90:0/874678984 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 -- 172.21.15.90:0/874678984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 msgr2=0x55926e74e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e74e8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 -- 172.21.15.90:0/874678984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 msgr2=0x55926e7511d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e7511d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.844+0000 7f5d0972c700 1 -- 172.21.15.90:0/874678984 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55926e5173b0 con 0x55926e744780 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.845+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e74e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:31.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.845+0000 7f5d0972c700 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 0x55926e744b60 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f5d000ca6d0 tx=0x7f5d00099a70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=10d5fe2adeebcc5a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.848+0000 7f5cef7fe700 1 -- 172.21.15.90:0/874678984 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d0009dec0 con 0x55926e744780 2024-04-06T14:15:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.848+0000 7f5cef7fe700 1 -- 172.21.15.90:0/874678984 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5d000a2510 con 0x55926e744780 2024-04-06T14:15:31.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.848+0000 7f5cef7fe700 1 -- 172.21.15.90:0/874678984 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5d000a2080 con 0x55926e744780 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 -- 172.21.15.90:0/874678984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 msgr2=0x55926e744b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 0x55926e744b60 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f5d000ca6d0 tx=0x7f5d00099a70 comp rx=0 tx=0).stop 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 -- 172.21.15.90:0/874678984 shutdown_connections 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e7511d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e744780 0x55926e744b60 secure :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f5d000ca6d0 tx=0x7f5d00099a70 comp rx=0 tx=0).stop 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/874678984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e74e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 -- 172.21.15.90:0/874678984 >> 172.21.15.90:0/874678984 conn(0x55926e6a36a0 msgr2=0x55926e73fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.853+0000 7f5d17b44b80 1 -- 172.21.15.90:0/874678984 shutdown_connections 2024-04-06T14:15:31.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.854+0000 7f5d17b44b80 1 -- 172.21.15.90:0/874678984 wait complete. 2024-04-06T14:15:31.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.854+0000 7f5d17b44b80 1 -- start start 2024-04-06T14:15:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.854+0000 7f5d17b44b80 1 Processor -- start 2024-04-06T14:15:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e7450a0 0x55926e74bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e763320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e631d80 con 0x55926e74c850 2024-04-06T14:15:31.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e631a20 con 0x55926e7450a0 2024-04-06T14:15:31.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e5196a0 con 0x55926e74ede0 2024-04-06T14:15:31.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d17b44b80 10 client.0 _ll_register_callbacks cb 0x55926e63b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:59008/0 (socket says 172.21.15.90:59008) 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.855+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3650024543 learned_addr learned my addr 172.21.15.90:0/3650024543 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.856+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e7450a0 msgr2=0x55926e74bdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.856+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e7450a0 0x55926e74bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.856+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3650024543 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 msgr2=0x55926e763320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.856+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e763320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.856+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3650024543 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5cf4004530 con 0x55926e74ede0 2024-04-06T14:15:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400fab0 tx=0x7f5cf4015000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cc528184c8239fdf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5cecff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:15:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cf4023040 con 0x55926e74ede0 2024-04-06T14:15:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5cf401c9b0 con 0x55926e74ede0 2024-04-06T14:15:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55926e656380 con 0x55926e74ede0 2024-04-06T14:15:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.857+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5cf4014040 con 0x55926e74ede0 2024-04-06T14:15:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 10 client.26641 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f5d041a6300 con 0x55926e74ede0 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55926e656380 con 0x55926e74ede0 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5cf402f070 con 0x55926e74ede0 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 10 client.26641 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 10 client.26641 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 20 client.26641 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7788d0 0x55926e77acf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 4 client.26641 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:15:31.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.858+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55926e68e700 con 0x55926e7788d0 2024-04-06T14:15:31.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.860+0000 7f5d0972c700 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7788d0 0x55926e77acf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.861+0000 7f5d0972c700 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7788d0 0x55926e77acf0 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=19fad42cdd14b84f server_cookie=824f4bf13eca97df in_seq=0 out_seq=0 2024-04-06T14:15:31.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.861+0000 7f5cecff9700 10 client.26641 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.863+0000 7f5cecff9700 1 -- 172.21.15.90:0/3650024543 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55926e68e700 con 0x55926e7788d0 2024-04-06T14:15:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.863+0000 7f5cecff9700 10 client.26641 handle_command_reply: tid=0 2024-04-06T14:15:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.863+0000 7f5d17b44b80 1 client.26641 shutdown 2024-04-06T14:15:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.863+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 msgr2=0x55926e74c310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.863+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400fab0 tx=0x7f5cf4015000 comp rx=0 tx=0).stop 2024-04-06T14:15:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 shutdown_connections 2024-04-06T14:15:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e763320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e7450a0 0x55926e74bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7788d0 0x55926e77acf0 crc :-1 s=READY pgs=183 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3650024543 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e74ede0 0x55926e74c310 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 >> 172.21.15.90:0/3650024543 conn(0x55926e6a36a0 msgr2=0x55926e73ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5cecff9700 0 client.26641 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 shutdown_connections 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3650024543 wait complete. 2024-04-06T14:15:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.864+0000 7f5d17b44b80 20 client.26641 trim_cache size 0 max 16384 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 Processor -- start 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 -- start start 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e763150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e73f8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e7753f0 con 0x55926e74ede0 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e737280 con 0x55926e74c850 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e763690 con 0x55926e7450a0 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e763150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:43786/0 (socket says 172.21.15.90:43786) 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d09f2d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e73f8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.888+0000 7f5d08f2b700 1 -- 172.21.15.90:0/1693186358 learned_addr learned my addr 172.21.15.90:0/1693186358 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 -- 172.21.15.90:0/1693186358 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 msgr2=0x55926e763150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e763150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 -- 172.21.15.90:0/1693186358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 msgr2=0x55926e73f8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e73f8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 -- 172.21.15.90:0/1693186358 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55926e746b50 con 0x55926e74c850 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d09f2d700 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e73f8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d0972c700 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e763150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400ae10 tx=0x7f5cf400a990 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6467a15b0d87e48a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5ceffff700 1 -- 172.21.15.90:0/1693186358 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cf4023070 con 0x55926e74c850 2024-04-06T14:15:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5ceffff700 1 -- 172.21.15.90:0/1693186358 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5cf4003ac0 con 0x55926e74c850 2024-04-06T14:15:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.889+0000 7f5ceffff700 1 -- 172.21.15.90:0/1693186358 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cf4014040 con 0x55926e74c850 2024-04-06T14:15:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- 172.21.15.90:0/1693186358 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 msgr2=0x55926e73f3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400ae10 tx=0x7f5cf400a990 comp rx=0 tx=0).stop 2024-04-06T14:15:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- 172.21.15.90:0/1693186358 shutdown_connections 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74ede0 0x55926e73f8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e763150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/1693186358 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74c850 0x55926e73f3a0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- 172.21.15.90:0/1693186358 >> 172.21.15.90:0/1693186358 conn(0x55926e6a36a0 msgr2=0x55926e762a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- 172.21.15.90:0/1693186358 shutdown_connections 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- 172.21.15.90:0/1693186358 wait complete. 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.890+0000 7f5d17b44b80 1 -- start start 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 Processor -- start 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e75c410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 0x55926e7c56d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e7c5c80 con 0x55926e74c850 2024-04-06T14:15:31.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e7c5df0 con 0x55926e74ede0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55926e7c5fb0 con 0x55926e7450a0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d17b44b80 10 client.0 _ll_register_callbacks cb 0x55926e63b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e75c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d08f2b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55016/0 (socket says 172.21.15.90:55016) 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3253851209 learned_addr learned my addr 172.21.15.90:0/3253851209 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d0972c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e75c410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:59026/0 (socket says 172.21.15.90:59026) 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.891+0000 7f5d09f2d700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 0x55926e7c56d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3253851209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 msgr2=0x55926e75c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e75c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 msgr2=0x55926e7c56d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 0x55926e7c56d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 -- 172.21.15.90:0/3253851209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5d0410caf0 con 0x55926e74c850 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d09f2d700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 0x55926e7c56d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:15:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d08f2b700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400fab0 tx=0x7f5cf4027ae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=494b2deb283bd8b5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5cecff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cf4023030 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5cf401c700 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.892+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55926e4e4cf0 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5cf4014040 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 10 client.17160 fetch_fsmap learned FSMap version 24 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f5d041a6300 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5cf4047020 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5cf402f070 con 0x55926e74c850 2024-04-06T14:15:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 10 client.17160 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 10 client.17160 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 20 client.17160 populate_metadata read hostname 'smithi090' 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7cd540 0x55926e7cf960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 4 client.17160 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.893+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55926e519ce0 con 0x55926e7cd540 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d0972c700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7cd540 0x55926e7cf960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5cecff9700 10 client.17160 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d0972c700 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7cd540 0x55926e7cf960 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d6489a4c17f4ea42 server_cookie=dcc997297aa36175 in_seq=0 out_seq=0 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5cecff9700 1 -- 172.21.15.90:0/3253851209 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55926e519ce0 con 0x55926e7cd540 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5cecff9700 10 client.17160 handle_command_reply: tid=0 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 client.17160 shutdown 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 msgr2=0x55926e75c950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 secure :-1 s=READY pgs=725 cs=0 l=1 rev1=1 crypto rx=0x7f5cf400fab0 tx=0x7f5cf4027ae0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 shutdown_connections 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55926e74ede0 0x55926e7c56d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55926e7cd540 0x55926e7cf960 crc :-1 s=READY pgs=184 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55926e7450a0 0x55926e75c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 --2- 172.21.15.90:0/3253851209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55926e74c850 0x55926e75c950 unknown :-1 s=CLOSED pgs=725 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 >> 172.21.15.90:0/3253851209 conn(0x55926e6a36a0 msgr2=0x55926e743770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.894+0000 7f5cecff9700 0 client.17160 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.895+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 shutdown_connections 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.895+0000 7f5d17b44b80 1 -- 172.21.15.90:0/3253851209 wait complete. 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.895+0000 7f5d17b44b80 20 client.17160 trim_cache size 0 max 16384 2024-04-06T14:15:31.900 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "9bc7d263-857d-4be7-b046-9594139e0583": { 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "9bc7d263-857d-4be7-b046-9594139e0583", 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:15:31.901 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:15:31.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 msgr2=0x7f5cf00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 0x7f5cf00a3750 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5d0419a8c0 tx=0x7f5cf4011040 comp rx=0 tx=0).stop 2024-04-06T14:15:31.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 msgr2=0x7f5d04199370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:15:31.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 secure :-1 s=READY pgs=724 cs=0 l=1 rev1=1 crypto rx=0x7f5d0000e9a0 tx=0x7f5d0000ecb0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 shutdown_connections 2024-04-06T14:15:31.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5cf00a1290 0x7f5cf00a3750 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5d0419dc50 0x7f5d0419e0d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5d040ff5e0 0x7f5d04199370 unknown :-1 s=CLOSED pgs=724 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 --2- 172.21.15.90:0/975234296 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5d04104700 0x7f5d041998b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:15:31.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 >> 172.21.15.90:0/975234296 conn(0x7f5d040761b0 msgr2=0x7f5d04102940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:15:31.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 shutdown_connections 2024-04-06T14:15:31.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:15:31.897+0000 7f5d0b990700 1 -- 172.21.15.90:0/975234296 wait complete. 2024-04-06T14:15:31.910 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:31 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:15:31.910 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:31 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:31.910 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:31 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:15:31.912 INFO:tasks.cephfs.filesystem:scrub status for tag:9bc7d263-857d-4be7-b046-9594139e0583 - {'path': '/', 'tag': '9bc7d263-857d-4be7-b046-9594139e0583', 'options': 'recursive,force'} 2024-04-06T14:15:32.148 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:31 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:15:32.148 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:31 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:32.148 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:31 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:15:32.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:31 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:15:32.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:31 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:32.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:31 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:15:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 585.0 s, 2084.9 tps, lat 46.665 ms stddev 55.330 2024-04-06T14:15:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:32 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:33.149 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:32 smithi134 ceph-mon[47549]: pgmap v315: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 271 KiB/s rd, 27 MiB/s wr, 1.79k op/s 2024-04-06T14:15:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:32 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:33.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:32 smithi105 ceph-mon[46969]: pgmap v315: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 271 KiB/s rd, 27 MiB/s wr, 1.79k op/s 2024-04-06T14:15:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:32 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:15:33.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:32 smithi090 ceph-mon[43420]: pgmap v315: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 271 KiB/s rd, 27 MiB/s wr, 1.79k op/s 2024-04-06T14:15:34.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:34 smithi134 ceph-mon[47549]: pgmap v316: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:34 smithi105 ceph-mon[46969]: pgmap v316: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:34.544 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:34 smithi090 ceph-mon[43420]: pgmap v316: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:36.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:35 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:15:36.252 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:35 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:15:36.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:35 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:15:37.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:36 smithi134 ceph-mon[47549]: pgmap v317: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 258 KiB/s rd, 24 MiB/s wr, 1.74k op/s 2024-04-06T14:15:37.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:36 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:15:37.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:36 smithi105 ceph-mon[46969]: pgmap v317: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 258 KiB/s rd, 24 MiB/s wr, 1.74k op/s 2024-04-06T14:15:37.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:36 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:15:37.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:36 smithi090 ceph-mon[43420]: pgmap v317: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 258 KiB/s rd, 24 MiB/s wr, 1.74k op/s 2024-04-06T14:15:37.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:36 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:15:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 590.0 s, 1458.4 tps, lat 69.559 ms stddev 89.104 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: pgmap v318: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 22 MiB/s wr, 1.61k op/s 2024-04-06T14:15:38.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.792 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: pgmap v318: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 22 MiB/s wr, 1.61k op/s 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.793 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.794 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:15:38.795 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:15:38.796 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.893 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: pgmap v318: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 22 MiB/s wr, 1.61k op/s 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi105 to 182.5M 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi105 to 191372697: error parsing value: Value '191372697' is below minimum 939524096 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.894 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:15:38.895 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:15:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:39 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:15:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:39 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:15:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:39 smithi134 ceph-mon[47549]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:15:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:39 smithi134 ceph-mon[47549]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:15:40.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:39 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:15:40.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:39 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:15:40.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:39 smithi105 ceph-mon[46969]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:15:40.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:39 smithi105 ceph-mon[46969]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:15:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:39 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi090 to 182.4M 2024-04-06T14:15:40.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:39 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi090 to 191351193: error parsing value: Value '191351193' is below minimum 939524096 2024-04-06T14:15:40.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:39 smithi090 ceph-mon[43420]: Adjusting osd_memory_target on smithi134 to 182.5M 2024-04-06T14:15:40.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:39 smithi090 ceph-mon[43420]: Unable to set osd_memory_target on smithi134 to 191379865: error parsing value: Value '191379865' is below minimum 939524096 2024-04-06T14:15:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:40 smithi134 ceph-mon[47549]: pgmap v319: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 18 MiB/s wr, 1.25k op/s 2024-04-06T14:15:41.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:40 smithi105 ceph-mon[46969]: pgmap v319: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 18 MiB/s wr, 1.25k op/s 2024-04-06T14:15:41.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:40 smithi090 ceph-mon[43420]: pgmap v319: 97 pgs: 97 active+clean; 6.7 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 237 KiB/s rd, 18 MiB/s wr, 1.25k op/s 2024-04-06T14:15:43.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 595.0 s, 1611.6 tps, lat 62.775 ms stddev 96.234 2024-04-06T14:15:43.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:42 smithi105 ceph-mon[46969]: pgmap v320: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 255 KiB/s rd, 26 MiB/s wr, 1.94k op/s 2024-04-06T14:15:43.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:42 smithi090 ceph-mon[43420]: pgmap v320: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 255 KiB/s rd, 26 MiB/s wr, 1.94k op/s 2024-04-06T14:15:43.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:42 smithi134 ceph-mon[47549]: pgmap v320: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 255 KiB/s rd, 26 MiB/s wr, 1.94k op/s 2024-04-06T14:15:45.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:44 smithi105 ceph-mon[46969]: pgmap v321: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:15:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:44 smithi090 ceph-mon[43420]: pgmap v321: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:15:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:44 smithi134 ceph-mon[47549]: pgmap v321: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:15:47.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:46 smithi105 ceph-mon[46969]: pgmap v322: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:15:47.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:46 smithi090 ceph-mon[43420]: pgmap v322: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:15:47.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:46 smithi134 ceph-mon[47549]: pgmap v322: 97 pgs: 97 active+clean; 6.8 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:15:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 600.0 s, 1943.4 tps, lat 51.188 ms stddev 68.135 2024-04-06T14:15:49.144 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:48 smithi134 ceph-mon[47549]: pgmap v323: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.46k op/s 2024-04-06T14:15:49.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:48 smithi105 ceph-mon[46969]: pgmap v323: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.46k op/s 2024-04-06T14:15:49.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:48 smithi090 ceph-mon[43420]: pgmap v323: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.46k op/s 2024-04-06T14:15:51.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:51 smithi090 ceph-mon[43420]: pgmap v324: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 18 MiB/s wr, 1.37k op/s 2024-04-06T14:15:51.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:51 smithi134 ceph-mon[47549]: pgmap v324: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 18 MiB/s wr, 1.37k op/s 2024-04-06T14:15:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:51 smithi105 ceph-mon[46969]: pgmap v324: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 18 MiB/s wr, 1.37k op/s 2024-04-06T14:15:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 605.0 s, 1687.6 tps, lat 58.836 ms stddev 63.350 2024-04-06T14:15:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:53 smithi090 ceph-mon[43420]: pgmap v325: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:53.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:53 smithi134 ceph-mon[47549]: pgmap v325: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:53.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:53 smithi105 ceph-mon[46969]: pgmap v325: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 21 MiB/s wr, 1.60k op/s 2024-04-06T14:15:54.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:54 smithi134 ceph-mon[47549]: pgmap v326: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 949 op/s 2024-04-06T14:15:54.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:54 smithi105 ceph-mon[46969]: pgmap v326: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 949 op/s 2024-04-06T14:15:54.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:54 smithi090 ceph-mon[43420]: pgmap v326: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 949 op/s 2024-04-06T14:15:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:57 smithi090 ceph-mon[43420]: pgmap v327: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 877 op/s 2024-04-06T14:15:57.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:57 smithi134 ceph-mon[47549]: pgmap v327: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 877 op/s 2024-04-06T14:15:57.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:57 smithi105 ceph-mon[46969]: pgmap v327: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 13 MiB/s wr, 877 op/s 2024-04-06T14:15:58.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 610.0 s, 2009.9 tps, lat 50.058 ms stddev 59.256 2024-04-06T14:15:59.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:15:58 smithi134 ceph-mon[47549]: pgmap v328: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 779 op/s 2024-04-06T14:15:59.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:15:58 smithi105 ceph-mon[46969]: pgmap v328: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 779 op/s 2024-04-06T14:15:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:15:58 smithi090 ceph-mon[43420]: pgmap v328: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 779 op/s 2024-04-06T14:16:01.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:00 smithi105 ceph-mon[46969]: pgmap v329: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 8.1 MiB/s wr, 542 op/s 2024-04-06T14:16:01.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:00 smithi090 ceph-mon[43420]: pgmap v329: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 8.1 MiB/s wr, 542 op/s 2024-04-06T14:16:01.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:00 smithi134 ceph-mon[47549]: pgmap v329: 97 pgs: 97 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 8.1 MiB/s wr, 542 op/s 2024-04-06T14:16:01.913 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:16:01.913 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 -- 172.21.15.90:0/3784062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28440ffcd0 msgr2=0x7f28441000b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 --2- 172.21.15.90:0/3784062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28440ffcd0 0x7f28441000b0 secure :-1 s=READY pgs=726 cs=0 l=1 rev1=1 crypto rx=0x7f283400aaa0 tx=0x7f283400adb0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 -- 172.21.15.90:0/3784062310 shutdown_connections 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 --2- 172.21.15.90:0/3784062310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2844104720 0x7f284410c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 --2- 172.21.15.90:0/3784062310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f28440ffcd0 0x7f28441000b0 unknown :-1 s=CLOSED pgs=726 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 --2- 172.21.15.90:0/3784062310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2844100680 0x7f2844104040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 -- 172.21.15.90:0/3784062310 >> 172.21.15.90:0/3784062310 conn(0x7f2844074b50 msgr2=0x7f2844074f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 -- 172.21.15.90:0/3784062310 shutdown_connections 2024-04-06T14:16:02.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.058+0000 7f2848e0a700 1 -- 172.21.15.90:0/3784062310 wait complete. 2024-04-06T14:16:02.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 Processor -- start 2024-04-06T14:16:02.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 -- start start 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28440ffcd0 0x7f28441971e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f284419bac0 0x7f284419bf40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f284419c540 con 0x7f2844104720 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f284419c6b0 con 0x7f284419bac0 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2848e0a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f284419c870 con 0x7f28440ffcd0 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:33886/0 (socket says 172.21.15.90:33886) 2024-04-06T14:16:02.062 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f284259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28440ffcd0 0x7f28441971e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 -- 172.21.15.90:0/2930973355 learned_addr learned my addr 172.21.15.90:0/2930973355 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2842d9d700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f284419bac0 0x7f284419bf40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 -- 172.21.15.90:0/2930973355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28440ffcd0 msgr2=0x7f28441971e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28440ffcd0 0x7f28441971e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 -- 172.21.15.90:0/2930973355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f284419bac0 msgr2=0x7f284419bf40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f284419bac0 0x7f284419bf40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.059+0000 7f2841d9b700 1 -- 172.21.15.90:0/2930973355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f283400a700 con 0x7f2844104720 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f2841d9b700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f283800e940 tx=0x7f283800ec50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f28337fe700 1 -- 172.21.15.90:0/2930973355 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2838009730 con 0x7f2844104720 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2844194d80 con 0x7f2844104720 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f28337fe700 1 -- 172.21.15.90:0/2930973355 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2838010430 con 0x7f2844104720 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f28337fe700 1 -- 172.21.15.90:0/2930973355 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f283800cea0 con 0x7f2844104720 2024-04-06T14:16:02.063 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.060+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2844195290 con 0x7f2844104720 2024-04-06T14:16:02.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.062+0000 7f28337fe700 1 -- 172.21.15.90:0/2930973355 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f2838010810 con 0x7f2844104720 2024-04-06T14:16:02.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.063+0000 7f28337fe700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 0x7f282c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.063+0000 7f28337fe700 1 -- 172.21.15.90:0/2930973355 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2838014070 con 0x7f2844104720 2024-04-06T14:16:02.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.063+0000 7f284259c700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 0x7f282c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.063+0000 7f284259c700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 0x7f282c0a3750 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f283400c6d0 tx=0x7f2834005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.064+0000 7f2854fbeb80 1 Processor -- start 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 -- start start 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd68a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 0x5638b2dd91d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2d24990 con 0x5638b2dcc780 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2d22350 con 0x5638b2dd6de0 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2d1dc40 con 0x5638b2dcd0a0 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:33890/0 (socket says 172.21.15.90:33890) 2024-04-06T14:16:02.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2841d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd68a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 -- 172.21.15.90:0/214455624 learned_addr learned my addr 172.21.15.90:0/214455624 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f2842d9d700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 0x5638b2dd91d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 -- 172.21.15.90:0/214455624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 msgr2=0x5638b2dd68a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd68a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 -- 172.21.15.90:0/214455624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 msgr2=0x5638b2dd91d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 0x5638b2dd91d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.065+0000 7f284259c700 1 -- 172.21.15.90:0/214455624 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5638b2b9f3b0 con 0x5638b2dcc780 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f2841d9b700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd68a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f2842d9d700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 0x5638b2dd91d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f284259c700 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f2834005cd0 tx=0x7f2834020000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29668f6b730bf0f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f2830ff9700 1 -- 172.21.15.90:0/214455624 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f283400ccc0 con 0x5638b2dcc780 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f2830ff9700 1 -- 172.21.15.90:0/214455624 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2834003960 con 0x5638b2dcc780 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.066+0000 7f2830ff9700 1 -- 172.21.15.90:0/214455624 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f283400e450 con 0x5638b2dcc780 2024-04-06T14:16:02.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- 172.21.15.90:0/214455624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 msgr2=0x5638b2dccb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 secure :-1 s=READY pgs=728 cs=0 l=1 rev1=1 crypto rx=0x7f2834005cd0 tx=0x7f2834020000 comp rx=0 tx=0).stop 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- 172.21.15.90:0/214455624 shutdown_connections 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dd6de0 0x5638b2dd91d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dccb60 unknown :-1 s=CLOSED pgs=728 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/214455624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd68a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- 172.21.15.90:0/214455624 >> 172.21.15.90:0/214455624 conn(0x5638b2d2b6a0 msgr2=0x5638b2dc7a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- 172.21.15.90:0/214455624 shutdown_connections 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- 172.21.15.90:0/214455624 wait complete. 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.067+0000 7f2854fbeb80 1 -- start start 2024-04-06T14:16:02.070 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 Processor -- start 2024-04-06T14:16:02.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd3ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd41f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2cb9d80 con 0x5638b2dcd0a0 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2cb9a20 con 0x5638b2dcc780 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2ba16a0 con 0x5638b2dd6de0 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2854fbeb80 10 client.0 _ll_register_callbacks cb 0x5638b2cc3470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35268/0 (socket says 172.21.15.90:35268) 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 -- 172.21.15.90:0/672170295 learned_addr learned my addr 172.21.15.90:0/672170295 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f2841d9b700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd3ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 -- 172.21.15.90:0/672170295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 msgr2=0x5638b2dd41f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd41f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 -- 172.21.15.90:0/672170295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcd0a0 msgr2=0x5638b2dd3ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd3ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.068+0000 7f284259c700 1 -- 172.21.15.90:0/672170295 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f283c011230 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f284259c700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f283400aaa0 tx=0x7f283402da40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=153f61669329f293 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2834005620 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2834026030 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5638b2cde380 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f283400e450 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2854fbeb80 10 client.26879 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f283401a040 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5638b2cde380 con 0x5638b2dcc780 2024-04-06T14:16:02.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.069+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f28340215b0 con 0x5638b2dcc780 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 10 client.26879 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 10 client.26879 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 20 client.26879 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2de6540 0x5638b2de8950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 4 client.26879 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5638b2d16700 con 0x5638b2de6540 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2841d9b700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2de6540 0x5638b2de8950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2829ffb700 10 client.26879 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.070+0000 7f2841d9b700 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2de6540 0x5638b2de8950 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=382512af3347f21f server_cookie=c185e823f6829bc1 in_seq=0 out_seq=0 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2829ffb700 1 -- 172.21.15.90:0/672170295 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5638b2d16700 con 0x5638b2de6540 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2829ffb700 10 client.26879 handle_command_reply: tid=0 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 client.26879 shutdown 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 msgr2=0x5638b2dd6830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f283400aaa0 tx=0x7f283402da40 comp rx=0 tx=0).stop 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 shutdown_connections 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2de6540 0x5638b2de8950 crc :-1 s=READY pgs=185 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd41f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2dd6830 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/672170295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcd0a0 0x5638b2dd3ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 >> 172.21.15.90:0/672170295 conn(0x5638b2d2b6a0 msgr2=0x5638b2dc6c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2829ffb700 0 client.26879 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 shutdown_connections 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 1 -- 172.21.15.90:0/672170295 wait complete. 2024-04-06T14:16:02.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.071+0000 7f2854fbeb80 20 client.26879 trim_cache size 0 max 16384 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 Processor -- start 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 -- start start 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddda00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd6360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2dcd480 con 0x5638b2dcc780 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2dcda10 con 0x5638b2dcd0a0 2024-04-06T14:16:02.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2dd50f0 con 0x5638b2dd6de0 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f284259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f284259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:33912/0 (socket says 172.21.15.90:33912) 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f284259c700 1 -- 172.21.15.90:0/3747210836 learned_addr learned my addr 172.21.15.90:0/3747210836 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.089+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddda00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2842d9d700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd6360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.095 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 -- 172.21.15.90:0/3747210836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 msgr2=0x5638b2dd6360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd6360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 -- 172.21.15.90:0/3747210836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 msgr2=0x5638b2dd3a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 -- 172.21.15.90:0/3747210836 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5638b2b4bb60 con 0x5638b2dcd0a0 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2842d9d700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd6360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f284259c700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddda00 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f28380ca5c0 tx=0x7f283801a040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=794f04ab003c9f1b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f28317fa700 1 -- 172.21.15.90:0/3747210836 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f283809ddb0 con 0x5638b2dcd0a0 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f28317fa700 1 -- 172.21.15.90:0/3747210836 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28380a2400 con 0x5638b2dcd0a0 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.090+0000 7f28317fa700 1 -- 172.21.15.90:0/3747210836 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28380996d0 con 0x5638b2dcd0a0 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3747210836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 msgr2=0x5638b2ddda00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddda00 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f28380ca5c0 tx=0x7f283801a040 comp rx=0 tx=0).stop 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3747210836 shutdown_connections 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dd6de0 0x5638b2dd6360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dcc780 0x5638b2dd3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3747210836 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddda00 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3747210836 >> 172.21.15.90:0/3747210836 conn(0x5638b2d2b6a0 msgr2=0x5638b2dd3340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3747210836 shutdown_connections 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3747210836 wait complete. 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- start start 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 Processor -- start 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddbbd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dd6de0 0x5638b2ddc110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.091+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2ddc7f0 con 0x5638b2dd6de0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2ddc960 con 0x5638b2dcc780 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2854fbeb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5638b2e3a350 con 0x5638b2dcd0a0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2854fbeb80 10 client.0 _ll_register_callbacks cb 0x5638b2cc3470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35288/0 (socket says 172.21.15.90:35288) 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2841d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddbbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 -- 172.21.15.90:0/3869086612 learned_addr learned my addr 172.21.15.90:0/3869086612 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2842d9d700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dd6de0 0x5638b2ddc110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 -- 172.21.15.90:0/3869086612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 msgr2=0x5638b2ddbbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddbbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 -- 172.21.15.90:0/3869086612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dd6de0 msgr2=0x5638b2ddc110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:02.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dd6de0 0x5638b2ddc110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 -- 172.21.15.90:0/3869086612 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f283401a040 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f284259c700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f2834021940 tx=0x7f2834021d10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ed9572172698857c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2829ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f283402d870 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2834026040 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.092+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2834003960 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2834021440 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 10 client.26885 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f2844194d80 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2834003960 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f283400e670 con 0x5638b2dcc780 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 10 client.26885 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:02.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 10 client.26885 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 20 client.26885 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2e3b5e0 0x5638b2e3da00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 4 client.26885 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5638b2b6ccf0 con 0x5638b2e3b5e0 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.093+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2e3b5e0 0x5638b2e3da00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2829ffb700 10 client.26885 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:02.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2841d9b700 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2e3b5e0 0x5638b2e3da00 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=40582e9c395895b1 server_cookie=d7cfa61524067093 in_seq=0 out_seq=0 2024-04-06T14:16:02.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2829ffb700 1 -- 172.21.15.90:0/3869086612 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x5638b2b6ccf0 con 0x5638b2e3b5e0 2024-04-06T14:16:02.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2829ffb700 10 client.26885 handle_command_reply: tid=0 2024-04-06T14:16:02.101 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 client.26885 shutdown 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 msgr2=0x5638b2ddd7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f2834021940 tx=0x7f2834021d10 comp rx=0 tx=0).stop 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 shutdown_connections 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5638b2dd6de0 0x5638b2ddc110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5638b2dcc780 0x5638b2ddd7a0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5638b2e3b5e0 0x5638b2e3da00 crc :-1 s=READY pgs=186 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 --2- 172.21.15.90:0/3869086612 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5638b2dcd0a0 0x5638b2ddbbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2829ffb700 0 client.26885 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 >> 172.21.15.90:0/3869086612 conn(0x5638b2d2b6a0 msgr2=0x5638b2dcc000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 shutdown_connections 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 1 -- 172.21.15.90:0/3869086612 wait complete. 2024-04-06T14:16:02.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.094+0000 7f2854fbeb80 20 client.26885 trim_cache size 0 max 16384 2024-04-06T14:16:02.103 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:02.103 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:16:02.103 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:16:02.103 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 msgr2=0x7f282c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 0x7f282c0a3750 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f283400c6d0 tx=0x7f2834005fb0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 msgr2=0x7f2844197720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 secure :-1 s=READY pgs=727 cs=0 l=1 rev1=1 crypto rx=0x7f283800e940 tx=0x7f283800ec50 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 shutdown_connections 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f282c0a1290 0x7f282c0a3750 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f284419bac0 0x7f284419bf40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f28440ffcd0 0x7f28441971e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 --2- 172.21.15.90:0/2930973355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2844104720 0x7f2844197720 unknown :-1 s=CLOSED pgs=727 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 >> 172.21.15.90:0/2930973355 conn(0x7f2844074b50 msgr2=0x7f28440fd5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.104 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 shutdown_connections 2024-04-06T14:16:02.105 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.095+0000 7f2848e0a700 1 -- 172.21.15.90:0/2930973355 wait complete. 2024-04-06T14:16:02.108 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:16:02.108 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:16:02.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 -- 172.21.15.90:0/533577389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 msgr2=0x7f99041127c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.339 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 --2- 172.21.15.90:0/533577389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 0x7f99041127c0 secure :-1 s=READY pgs=729 cs=0 l=1 rev1=1 crypto rx=0x7f98fc00b0a0 tx=0x7f98fc00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 -- 172.21.15.90:0/533577389 shutdown_connections 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 --2- 172.21.15.90:0/533577389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9904112d00 0x7f99041150f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 --2- 172.21.15.90:0/533577389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f9904109b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 --2- 172.21.15.90:0/533577389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 0x7f99041127c0 unknown :-1 s=CLOSED pgs=729 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.339+0000 7f990a523700 1 -- 172.21.15.90:0/533577389 >> 172.21.15.90:0/533577389 conn(0x7f990406bd30 msgr2=0x7f990406c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.340 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.340+0000 7f990a523700 1 -- 172.21.15.90:0/533577389 shutdown_connections 2024-04-06T14:16:02.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.344+0000 7f990a523700 1 -- 172.21.15.90:0/533577389 wait complete. 2024-04-06T14:16:02.345 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.345+0000 7f990a523700 1 Processor -- start 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.350+0000 7f990a523700 1 -- start start 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 0x7f99041a6660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9904112d00 0x7f99041aa9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99041aaff0 con 0x7f990410a070 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99041ab160 con 0x7f9904109750 2024-04-06T14:16:02.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.351+0000 7f990a523700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f99041ab320 con 0x7f9904112d00 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:35302/0 (socket says 172.21.15.90:35302) 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 -- 172.21.15.90:0/2524848623 learned_addr learned my addr 172.21.15.90:0/2524848623 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 -- 172.21.15.90:0/2524848623 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9904112d00 msgr2=0x7f99041aa9b0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9904112d00 0x7f99041aa9b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 -- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 msgr2=0x7f99041a6660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 0x7f99041a6660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.352+0000 7f9903fff700 1 -- 172.21.15.90:0/2524848623 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98fc009d00 con 0x7f9904109750 2024-04-06T14:16:02.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.358+0000 7f9903fff700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f98f400e9a0 tx=0x7f98f400ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.373+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f4009790 con 0x7f9904109750 2024-04-06T14:16:02.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.373+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f98f400cbd0 con 0x7f9904109750 2024-04-06T14:16:02.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.373+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f4018680 con 0x7f9904109750 2024-04-06T14:16:02.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.376+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f99041ab600 con 0x7f9904109750 2024-04-06T14:16:02.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.376+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f990419d840 con 0x7f9904109750 2024-04-06T14:16:02.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.382+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f98f00052f0 con 0x7f9904109750 2024-04-06T14:16:02.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.385+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f98f400cd40 con 0x7f9904109750 2024-04-06T14:16:02.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.386+0000 7f99017fa700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 0x7f98ec0a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:02.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.386+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f98f4014070 con 0x7f9904109750 2024-04-06T14:16:02.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.389+0000 7f99037fe700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 0x7f98ec0a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:02.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.390+0000 7f99037fe700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 0x7f98ec0a3920 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f98fc00b070 tx=0x7f98fc00b700 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:02.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.400+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f98f4099e40 con 0x7f9904109750 2024-04-06T14:16:02.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.917+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f98f0005160 con 0x7f9904109750 2024-04-06T14:16:02.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.925+0000 7f99017fa700 1 -- 172.21.15.90:0/2524848623 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f98f40999d0 con 0x7f9904109750 2024-04-06T14:16:02.944 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:16:02.944 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:16:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.951+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 msgr2=0x7f98ec0a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.951+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 0x7f98ec0a3920 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f98fc00b070 tx=0x7f98fc00b700 comp rx=0 tx=0).stop 2024-04-06T14:16:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.951+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 msgr2=0x7f99041a6120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:02.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.951+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f98f400e9a0 tx=0x7f98f400ed60 comp rx=0 tx=0).stop 2024-04-06T14:16:02.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 shutdown_connections 2024-04-06T14:16:02.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f98ec0a1460 0x7f98ec0a3920 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.970 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9904112d00 0x7f99041aa9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9904109750 0x7f99041a6120 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 --2- 172.21.15.90:0/2524848623 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f990410a070 0x7f99041a6660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:02.971 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.969+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 >> 172.21.15.90:0/2524848623 conn(0x7f990406bd30 msgr2=0x7f990406f960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:02.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.975+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 shutdown_connections 2024-04-06T14:16:02.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:02.976+0000 7f990a523700 1 -- 172.21.15.90:0/2524848623 wait complete. 2024-04-06T14:16:02.997 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:16:03.006 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:03.006 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:16:03.029 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 615.0 s, 988.7 tps, lat 86.366 ms stddev 117.834 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 -- 172.21.15.90:0/2311327787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0100820 msgr2=0x7f0fe0104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/2311327787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0100820 0x7f0fe0104840 secure :-1 s=READY pgs=730 cs=0 l=1 rev1=1 crypto rx=0x7f0fd0009990 tx=0x7f0fd0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 -- 172.21.15.90:0/2311327787 shutdown_connections 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/2311327787 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe0104f70 0x7f0fe010caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/2311327787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe0100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/2311327787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0100820 0x7f0fe0104840 unknown :-1 s=CLOSED pgs=730 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 -- 172.21.15.90:0/2311327787 >> 172.21.15.90:0/2311327787 conn(0x7f0fe0076d50 msgr2=0x7f0fe0077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 -- 172.21.15.90:0/2311327787 shutdown_connections 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.246+0000 7f0fe7e09700 1 -- 172.21.15.90:0/2311327787 wait complete. 2024-04-06T14:16:03.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 Processor -- start 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 -- start start 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe019a9c0 0x7f0fe019ae40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fe019b440 con 0x7f0fe0104f70 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fe019b5b0 con 0x7f0fe00ffe70 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe7e09700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fe019b770 con 0x7f0fe019a9c0 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:44990/0 (socket says 172.21.15.90:44990) 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60576/0 (socket says 172.21.15.90:60576) 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/1895819162 learned_addr learned my addr 172.21.15.90:0/1895819162 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.247+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe019a9c0 0x7f0fe019ae40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/1895819162 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe019a9c0 msgr2=0x7f0fe019ae40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe019a9c0 0x7f0fe019ae40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 msgr2=0x7f0fe0196620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/1895819162 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fdc009520 con 0x7f0fe00ffe70 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc00eb30 tx=0x7f0fdc00ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fd6ffd700 1 -- 172.21.15.90:0/1895819162 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fdc009880 con 0x7f0fe00ffe70 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0fd00095f0 con 0x7f0fe00ffe70 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fd6ffd700 1 -- 172.21.15.90:0/1895819162 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0fdc0044e0 con 0x7f0fe00ffe70 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fd6ffd700 1 -- 172.21.15.90:0/1895819162 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fdc003d80 con 0x7f0fe00ffe70 2024-04-06T14:16:03.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.248+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0fe01a3070 con 0x7f0fe00ffe70 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.250+0000 7f0fd6ffd700 1 -- 172.21.15.90:0/1895819162 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0fdc003f20 con 0x7f0fe00ffe70 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.250+0000 7f0fd6ffd700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 0x7f0fcc0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.251+0000 7f0fd6ffd700 1 -- 172.21.15.90:0/1895819162 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0fdc014070 con 0x7f0fe00ffe70 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.251+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 0x7f0fcc0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.251+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 0x7f0fcc0a3750 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f0fd000b2e0 tx=0x7f0fd0005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 Processor -- start 2024-04-06T14:16:03.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 -- start start 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 0x5652173bbb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173c7ff0 0x5652173ca3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x565217313990 con 0x5652173bc0a0 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x565217311350 con 0x5652173bb780 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56521730cc40 con 0x5652173c7ff0 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60582/0 (socket says 172.21.15.90:60582) 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 -- 172.21.15.90:0/1125474236 learned_addr learned my addr 172.21.15.90:0/1125474236 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 0x5652173bbb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173c7ff0 0x5652173ca3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 -- 172.21.15.90:0/1125474236 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173c7ff0 msgr2=0x5652173ca3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173c7ff0 0x5652173ca3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 -- 172.21.15.90:0/1125474236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 msgr2=0x5652173bbb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 0x5652173bbb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe53a4700 1 -- 172.21.15.90:0/1125474236 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56521718e3b0 con 0x5652173bc0a0 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.252+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 0x5652173bbb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.253+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f0fd0015000 tx=0x7f0fd0020000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=abcfaa85ee4c9a29 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.253+0000 7f0fcbfff700 1 -- 172.21.15.90:0/1125474236 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fd002b070 con 0x5652173bc0a0 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.253+0000 7f0fcbfff700 1 -- 172.21.15.90:0/1125474236 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0fd001e040 con 0x5652173bc0a0 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.253+0000 7f0fcbfff700 1 -- 172.21.15.90:0/1125474236 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fd00048e0 con 0x5652173bc0a0 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/1125474236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 msgr2=0x5652173c7ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 secure :-1 s=READY pgs=731 cs=0 l=1 rev1=1 crypto rx=0x7f0fd0015000 tx=0x7f0fd0020000 comp rx=0 tx=0).stop 2024-04-06T14:16:03.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/1125474236 shutdown_connections 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173c7ff0 0x5652173ca3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173bb780 0x5652173bbb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/1125474236 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173c7ab0 unknown :-1 s=CLOSED pgs=731 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/1125474236 >> 172.21.15.90:0/1125474236 conn(0x56521731a6a0 msgr2=0x5652173b6a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/1125474236 shutdown_connections 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.254+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/1125474236 wait complete. 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 -- start start 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 Processor -- start 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173c5340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c5880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652172a8d80 con 0x5652173bb780 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652172a8a20 con 0x5652173c7ff0 2024-04-06T14:16:03.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652171906a0 con 0x5652173bc0a0 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.255+0000 7f0ff3fbdb80 10 client.0 _ll_register_callbacks cb 0x5652172b2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60586/0 (socket says 172.21.15.90:60586) 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2012111119 learned_addr learned my addr 172.21.15.90:0/2012111119 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c5880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2012111119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 msgr2=0x5652173c5340 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173c5340 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 msgr2=0x5652173c5880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c5880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2012111119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0fe01a3070 con 0x5652173bb780 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c5880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc00c5b0 tx=0x7f0fdc099dd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f1b50c9b03ccbfdf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fc97fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fdc09e170 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0fdc0a27c0 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5652172cd380 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0fdc00cdd0 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0ff3fbdb80 10 client.17193 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f0fe010d360 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.256+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5652172cd380 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0fdc0a2330 con 0x5652173bb780 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 10 client.17193 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 10 client.17193 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 20 client.17193 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173d7cb0 0x5652173da0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 4 client.17193 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x565217305700 con 0x5652173d7cb0 2024-04-06T14:16:03.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173d7cb0 0x5652173da0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0fc97fa700 10 client.17193 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.257+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173d7cb0 0x5652173da0d0 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=541491c7d48cd8a8 server_cookie=e4d2ff181d8ac1ca in_seq=0 out_seq=0 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.258+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2012111119 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x565217305700 con 0x5652173d7cb0 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.258+0000 7f0fc97fa700 10 client.17193 handle_command_reply: tid=0 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.258+0000 7f0ff3fbdb80 1 client.17193 shutdown 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 msgr2=0x5652173c4e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 secure :-1 s=READY pgs=732 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc00c5b0 tx=0x7f0fdc099dd0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 shutdown_connections 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173d7cb0 0x5652173da0d0 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c5880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173c4e00 unknown :-1 s=CLOSED pgs=732 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2012111119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173c5340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 >> 172.21.15.90:0/2012111119 conn(0x56521731a6a0 msgr2=0x5652173b5c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0fc97fa700 0 client.17193 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:03.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 shutdown_connections 2024-04-06T14:16:03.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2012111119 wait complete. 2024-04-06T14:16:03.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.260+0000 7f0ff3fbdb80 20 client.17193 trim_cache size 0 max 16384 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.273+0000 7f0ff3fbdb80 1 Processor -- start 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.273+0000 7f0ff3fbdb80 1 -- start start 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 0x5652173b63a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c2080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652173c3350 con 0x5652173bc0a0 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652173bcfe0 con 0x5652173c7ff0 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5652173d7a40 con 0x5652173bb780 2024-04-06T14:16:03.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 0x5652173b63a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60598/0 (socket says 172.21.15.90:60598) 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 -- 172.21.15.90:0/3979272599 learned_addr learned my addr 172.21.15.90:0/3979272599 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c2080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 -- 172.21.15.90:0/3979272599 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 msgr2=0x5652173b63a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 0x5652173b63a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 -- 172.21.15.90:0/3979272599 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 msgr2=0x5652173c2080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c2080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 -- 172.21.15.90:0/3979272599 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56521713ab60 con 0x5652173bc0a0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 0x5652173b63a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c2080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f0fd000fa20 tx=0x7f0fd0009ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=895cea5d40cbae53 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.274+0000 7f0fd4ff9700 1 -- 172.21.15.90:0/3979272599 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fd002b040 con 0x5652173bc0a0 2024-04-06T14:16:03.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0fd4ff9700 1 -- 172.21.15.90:0/3979272599 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0fd0005600 con 0x5652173bc0a0 2024-04-06T14:16:03.277 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0fd4ff9700 1 -- 172.21.15.90:0/3979272599 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fd0024cc0 con 0x5652173bc0a0 2024-04-06T14:16:03.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/3979272599 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 msgr2=0x5652173b68e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 secure :-1 s=READY pgs=733 cs=0 l=1 rev1=1 crypto rx=0x7f0fd000fa20 tx=0x7f0fd0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/3979272599 shutdown_connections 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173c2080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bb780 0x5652173b63a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/3979272599 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bc0a0 0x5652173b68e0 unknown :-1 s=CLOSED pgs=733 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/3979272599 >> 172.21.15.90:0/3979272599 conn(0x56521731a6a0 msgr2=0x5652173c00f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/3979272599 shutdown_connections 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.275+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/3979272599 wait complete. 2024-04-06T14:16:03.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 -- start start 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 Processor -- start 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173cd130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173cd670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56521742aae0 con 0x5652173bb780 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56521742ac20 con 0x5652173c7ff0 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56521742ad60 con 0x5652173bc0a0 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0ff3fbdb80 10 client.0 _ll_register_callbacks cb 0x5652172b2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0fe5ba5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60602/0 (socket says 172.21.15.90:60602) 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.276+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2104587667 learned_addr learned my addr 172.21.15.90:0/2104587667 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173cd130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe63a6700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173cd670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2104587667 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 msgr2=0x5652173cd130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173cd130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 msgr2=0x5652173cd670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173cd670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 -- 172.21.15.90:0/2104587667 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0fe010d360 con 0x5652173bb780 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fe5ba5700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc096db0 tx=0x7f0fdc0a2540 comp rx=0 tx=0).ready entity=mon.0 client_cookie=adfbc56120ca70b1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fc97fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fdc099a60 con 0x5652173bb780 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0fdc099830 con 0x5652173bb780 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0fd001e040 con 0x5652173bb780 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0fdc0a27c0 con 0x5652173bb780 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0ff3fbdb80 10 client.17199 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f0fd80073e0 con 0x5652173bb780 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.277+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0fdc08c020 con 0x5652173bb780 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f0fdc0992a0 con 0x5652173bb780 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 10 client.17199 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 10 client.17199 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 20 client.17199 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173bdc10 0x5652173be020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 4 client.17199 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x56521715bcf0 con 0x5652173bdc10 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.278+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173bdc10 0x5652173be020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0fe53a4700 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173bdc10 0x5652173be020 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=7f00a8a90c814804 server_cookie=d53f2bfa77c75123 in_seq=0 out_seq=0 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0fc97fa700 10 client.17199 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0fc97fa700 1 -- 172.21.15.90:0/2104587667 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x56521715bcf0 con 0x5652173bdc10 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0fc97fa700 10 client.17199 handle_command_reply: tid=0 2024-04-06T14:16:03.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 client.17199 shutdown 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 msgr2=0x5652173ccbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 secure :-1 s=READY pgs=734 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc096db0 tx=0x7f0fdc0a2540 comp rx=0 tx=0).stop 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 shutdown_connections 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x5652173bdc10 0x5652173be020 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5652173c7ff0 0x5652173cd670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5652173bb780 0x5652173ccbf0 unknown :-1 s=CLOSED pgs=734 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 --2- 172.21.15.90:0/2104587667 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5652173bc0a0 0x5652173cd130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 >> 172.21.15.90:0/2104587667 conn(0x56521731a6a0 msgr2=0x5652173c36d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0fc97fa700 0 client.17199 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 shutdown_connections 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 1 -- 172.21.15.90:0/2104587667 wait complete. 2024-04-06T14:16:03.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.279+0000 7f0ff3fbdb80 20 client.17199 trim_cache size 0 max 16384 2024-04-06T14:16:03.286 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:03.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 msgr2=0x7f0fcc0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 0x7f0fcc0a3750 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f0fd000b2e0 tx=0x7f0fd0005960 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 msgr2=0x7f0fe01960e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f0fdc00eb30 tx=0x7f0fdc00ee40 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 shutdown_connections 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0fcc0a1290 0x7f0fcc0a3750 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0fe019a9c0 0x7f0fe019ae40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0fe00ffe70 0x7f0fe01960e0 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 --2- 172.21.15.90:0/1895819162 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0fe0104f70 0x7f0fe0196620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 >> 172.21.15.90:0/1895819162 conn(0x7f0fe0076d50 msgr2=0x7f0fe00fde60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 shutdown_connections 2024-04-06T14:16:03.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.280+0000 7f0fe7e09700 1 -- 172.21.15.90:0/1895819162 wait complete. 2024-04-06T14:16:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:03 smithi090 ceph-mon[43420]: pgmap v330: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 12 MiB/s wr, 815 op/s 2024-04-06T14:16:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:03 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2524848623' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:03.293 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:03.294 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:16:03.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:03 smithi134 ceph-mon[47549]: pgmap v330: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 12 MiB/s wr, 815 op/s 2024-04-06T14:16:03.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:03 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2524848623' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:03.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 -- 172.21.15.90:0/150524930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c069ca0 msgr2=0x7fe28c105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 --2- 172.21.15.90:0/150524930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c069ca0 0x7fe28c105970 secure :-1 s=READY pgs=735 cs=0 l=1 rev1=1 crypto rx=0x7fe2800099c0 tx=0x7fe280009cd0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 -- 172.21.15.90:0/150524930 shutdown_connections 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 --2- 172.21.15.90:0/150524930 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c1060a0 0x7fe28c10dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 --2- 172.21.15.90:0/150524930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 0x7fe28c0696d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 --2- 172.21.15.90:0/150524930 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c069ca0 0x7fe28c105970 unknown :-1 s=CLOSED pgs=735 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 -- 172.21.15.90:0/150524930 >> 172.21.15.90:0/150524930 conn(0x7fe28c076d40 msgr2=0x7fe28c077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 -- 172.21.15.90:0/150524930 shutdown_connections 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.486+0000 7fe292814700 1 -- 172.21.15.90:0/150524930 wait complete. 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 Processor -- start 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 -- start start 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 0x7fe28c1971f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c19bad0 0x7fe28c19bf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe28c19c550 con 0x7fe28c1060a0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe28c19c6c0 con 0x7fe28c0692f0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe292814700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe28c19c880 con 0x7fe28c19bad0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60626/0 (socket says 172.21.15.90:60626) 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 -- 172.21.15.90:0/3548846257 learned_addr learned my addr 172.21.15.90:0/3548846257 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28bfff700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 0x7fe28c1971f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe290db1700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c19bad0 0x7fe28c19bf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 -- 172.21.15.90:0/3548846257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c19bad0 msgr2=0x7fe28c19bf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c19bad0 0x7fe28c19bf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 -- 172.21.15.90:0/3548846257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 msgr2=0x7fe28c1971f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 0x7fe28c1971f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.487+0000 7fe28b7fe700 1 -- 172.21.15.90:0/3548846257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe2800095f0 con 0x7fe28c1060a0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7fe280009fd0 tx=0x7fe28000f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe2897fa700 1 -- 172.21.15.90:0/3548846257 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe28001c070 con 0x7fe28c1060a0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe28c1a3d80 con 0x7fe28c1060a0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe2897fa700 1 -- 172.21.15.90:0/3548846257 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe28000fd80 con 0x7fe28c1060a0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe2897fa700 1 -- 172.21.15.90:0/3548846257 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe280004970 con 0x7fe28c1060a0 2024-04-06T14:16:03.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.488+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe28c1a4140 con 0x7fe28c1060a0 2024-04-06T14:16:03.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 Processor -- start 2024-04-06T14:16:03.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 -- start start 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a084feeb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fef0a0 0x55a084ff88a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084f46990 con 0x55a084ff8de0 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084f44350 con 0x55a084fef0a0 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084f3fc40 con 0x55a084fee780 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.492+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60634/0 (socket says 172.21.15.90:60634) 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 -- 172.21.15.90:0/1195857370 learned_addr learned my addr 172.21.15.90:0/1195857370 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe290db1700 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a084feeb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28bfff700 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fef0a0 0x55a084ff88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 -- 172.21.15.90:0/1195857370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 msgr2=0x55a084feeb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a084feeb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 -- 172.21.15.90:0/1195857370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fef0a0 msgr2=0x55a084ff88a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fef0a0 0x55a084ff88a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 -- 172.21.15.90:0/1195857370 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a084dc13b0 con 0x55a084ff8de0 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7fe2800098e0 tx=0x7fe28000fba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29b37983e5105a3b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe2727fc700 1 -- 172.21.15.90:0/1195857370 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe280030070 con 0x55a084ff8de0 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe2727fc700 1 -- 172.21.15.90:0/1195857370 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe280031040 con 0x55a084ff8de0 2024-04-06T14:16:03.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.493+0000 7fe2727fc700 1 -- 172.21.15.90:0/1195857370 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe280036460 con 0x55a084ff8de0 2024-04-06T14:16:03.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1195857370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 msgr2=0x55a084ffb1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 secure :-1 s=READY pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7fe2800098e0 tx=0x7fe28000fba0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1195857370 shutdown_connections 2024-04-06T14:16:03.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084ff8de0 0x55a084ffb1d0 secure :-1 s=CLOSED pgs=737 cs=0 l=1 rev1=1 crypto rx=0x7fe2800098e0 tx=0x7fe28000fba0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a084feeb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1195857370 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fef0a0 0x55a084ff88a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.494+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1195857370 >> 172.21.15.90:0/1195857370 conn(0x55a084f4d6a0 msgr2=0x55a084fe9a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.495+0000 7fe2897fa700 1 -- 172.21.15.90:0/3548846257 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fe280020070 con 0x7fe28c1060a0 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.495+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1195857370 shutdown_connections 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.495+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1195857370 wait complete. 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe2897fa700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 0x7fe2740a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe29e9c8b80 1 -- start start 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe2897fa700 1 -- 172.21.15.90:0/3548846257 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe2800d0370 con 0x7fe28c1060a0 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe29e9c8b80 1 Processor -- start 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe28bfff700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 0x7fe2740a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fee780 0x55a084ff84e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.496+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 0x55a084ff21f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084edbd80 con 0x55a084fef0a0 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084edba20 con 0x55a084fee780 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084dc36a0 con 0x55a084ff1d70 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe29e9c8b80 10 client.0 _ll_register_callbacks cb 0x55a084ee5470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60650/0 (socket says 172.21.15.90:60650) 2024-04-06T14:16:03.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28b7fe700 1 -- 172.21.15.90:0/1722500587 learned_addr learned my addr 172.21.15.90:0/1722500587 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28bfff700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fee780 0x55a084ff84e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 0x55a084ff21f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 -- 172.21.15.90:0/1722500587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fee780 msgr2=0x55a084ff84e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fee780 0x55a084ff84e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 -- 172.21.15.90:0/1722500587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 msgr2=0x55a084ff1830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 -- 172.21.15.90:0/1722500587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe28c1a4140 con 0x55a084ff1d70 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe28bfff700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 0x7fe2740a3660 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fe27c012a40 tx=0x7fe27c009bf0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe290db1700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 0x55a084ff21f0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fe284009fd0 tx=0x7fe284004040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ee157b5550562d91 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe26b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.497+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe284004db0 con 0x55a084ff1d70 2024-04-06T14:16:03.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a084f00380 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe284007500 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe2840076c0 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 10 client.26692 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a084d6db60 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55a084f00380 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe284016800 con 0x55a084ff1d70 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 10 client.26692 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 10 client.26692 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 20 client.26692 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085011290 0x55a0850136b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 4 client.26692 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.498+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a084f38700 con 0x55a085011290 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.499+0000 7fe28bfff700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085011290 0x55a0850136b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.499+0000 7fe26b7fe700 10 client.26692 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:03.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.499+0000 7fe28bfff700 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085011290 0x55a0850136b0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=9286a263c0067ec0 server_cookie=60a5eb0257399ff5 in_seq=0 out_seq=0 2024-04-06T14:16:03.504 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:03 smithi105 ceph-mon[46969]: pgmap v330: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 12 MiB/s wr, 815 op/s 2024-04-06T14:16:03.504 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:03 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2524848623' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:03.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.499+0000 7fe26b7fe700 1 -- 172.21.15.90:0/1722500587 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a084f38700 con 0x55a085011290 2024-04-06T14:16:03.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe26b7fe700 10 client.26692 handle_command_reply: tid=0 2024-04-06T14:16:03.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 client.26692 shutdown 2024-04-06T14:16:03.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 msgr2=0x55a084ff21f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 0x55a084ff21f0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fe284009fd0 tx=0x7fe284004040 comp rx=0 tx=0).stop 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 shutdown_connections 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085011290 0x55a0850136b0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084ff1d70 0x55a084ff21f0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe26b7fe700 0 client.26692 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084fee780 0x55a084ff84e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/1722500587 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a084ff1830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 >> 172.21.15.90:0/1722500587 conn(0x55a084f4d6a0 msgr2=0x55a084fe8c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 shutdown_connections 2024-04-06T14:16:03.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/1722500587 wait complete. 2024-04-06T14:16:03.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.500+0000 7fe29e9c8b80 20 client.26692 trim_cache size 0 max 16384 2024-04-06T14:16:03.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 Processor -- start 2024-04-06T14:16:03.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 -- start start 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fef0a0 0x55a084ff0710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a084fff3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084ff00b0 con 0x55a084fee780 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084ff8b80 con 0x55a084ff1d70 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a084ff6a50 con 0x55a084fef0a0 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60664/0 (socket says 172.21.15.90:60664) 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 -- 172.21.15.90:0/2386488642 learned_addr learned my addr 172.21.15.90:0/2386488642 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fef0a0 0x55a084ff0710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe290db1700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a084fff3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 -- 172.21.15.90:0/2386488642 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fef0a0 msgr2=0x55a084ff0710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fef0a0 0x55a084ff0710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 -- 172.21.15.90:0/2386488642 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 msgr2=0x55a084fff3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a084fff3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe28bfff700 1 -- 172.21.15.90:0/2386488642 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a084f05ea0 con 0x55a084fee780 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.518+0000 7fe290db1700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a084fff3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe28bfff700 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fe27c0123e0 tx=0x7fe27c023000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc58eb9792460264 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe272ffd700 1 -- 172.21.15.90:0/2386488642 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe27c028040 con 0x55a084fee780 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe272ffd700 1 -- 172.21.15.90:0/2386488642 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe27c012410 con 0x55a084fee780 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe272ffd700 1 -- 172.21.15.90:0/2386488642 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe27c0237b0 con 0x55a084fee780 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/2386488642 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 msgr2=0x55a084ff5fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.519+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 secure :-1 s=READY pgs=738 cs=0 l=1 rev1=1 crypto rx=0x7fe27c0123e0 tx=0x7fe27c023000 comp rx=0 tx=0).stop 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/2386488642 shutdown_connections 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a084fff3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fee780 0x55a084ff5fe0 unknown :-1 s=CLOSED pgs=738 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/2386488642 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fef0a0 0x55a084ff0710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/2386488642 >> 172.21.15.90:0/2386488642 conn(0x55a084f4d6a0 msgr2=0x55a084ff10c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/2386488642 shutdown_connections 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/2386488642 wait complete. 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- start start 2024-04-06T14:16:03.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 Processor -- start 2024-04-06T14:16:03.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a08500dc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a08500a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a08500aa10 con 0x55a084fef0a0 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a08500ab80 con 0x55a084ff1d70 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a08500ad40 con 0x55a084fee780 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.520+0000 7fe29e9c8b80 10 client.0 _ll_register_callbacks cb 0x55a084ee5470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54380/0 (socket says 172.21.15.90:54380) 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe290db1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a08500a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 -- 172.21.15.90:0/4169422744 learned_addr learned my addr 172.21.15.90:0/4169422744 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a08500dc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 -- 172.21.15.90:0/4169422744 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 msgr2=0x55a08500a340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a08500a340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 -- 172.21.15.90:0/4169422744 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 msgr2=0x55a08500dc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a08500dc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 -- 172.21.15.90:0/4169422744 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe27c012150 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe290db1700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a08500a340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe28bfff700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fe27c005e70 tx=0x7fe27c012f10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ee84bc38deb35a46 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe26b7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe27c02b880 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a084d8ecf0 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe27c02b9e0 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe27c02bbe0 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe29e9c8b80 10 client.26698 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.521+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a084dc5cd0 con 0x55a084fee780 2024-04-06T14:16:03.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe27c040020 con 0x55a084fee780 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe27c031020 con 0x55a084fee780 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 10 client.26698 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 10 client.26698 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 20 client.26698 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085000b80 0x55a085002fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 4 client.26698 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55a084dc3ce0 con 0x55a085000b80 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085000b80 0x55a085002fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe26b7fe700 10 client.26698 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.522+0000 7fe28b7fe700 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085000b80 0x55a085002fa0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c0db088cd4b3b46c server_cookie=9fc9dcc87c56dae7 in_seq=0 out_seq=0 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.523+0000 7fe26b7fe700 1 -- 172.21.15.90:0/4169422744 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55a084dc3ce0 con 0x55a085000b80 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.523+0000 7fe26b7fe700 10 client.26698 handle_command_reply: tid=0 2024-04-06T14:16:03.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.524+0000 7fe29e9c8b80 1 client.26698 shutdown 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.524+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 msgr2=0x55a08500d720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.524+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fe27c005e70 tx=0x7fe27c012f10 comp rx=0 tx=0).stop 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 shutdown_connections 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a085000b80 0x55a085002fa0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a084ff1d70 0x55a08500a340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a084fee780 0x55a08500d720 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 --2- 172.21.15.90:0/4169422744 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a084fef0a0 0x55a08500dc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 >> 172.21.15.90:0/4169422744 conn(0x55a084f4d6a0 msgr2=0x55a084ff0fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe26b7fe700 0 client.26698 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 shutdown_connections 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 1 -- 172.21.15.90:0/4169422744 wait complete. 2024-04-06T14:16:03.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.525+0000 7fe29e9c8b80 20 client.26698 trim_cache size 0 max 16384 2024-04-06T14:16:03.531 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:03.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 msgr2=0x7fe2740a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 0x7fe2740a3660 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7fe27c012a40 tx=0x7fe27c009bf0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 msgr2=0x7fe28c197730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 secure :-1 s=READY pgs=736 cs=0 l=1 rev1=1 crypto rx=0x7fe280009fd0 tx=0x7fe28000f690 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 shutdown_connections 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe2740a11a0 0x7fe2740a3660 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe28c19bad0 0x7fe28c19bf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe28c0692f0 0x7fe28c1971f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 --2- 172.21.15.90:0/3548846257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe28c1060a0 0x7fe28c197730 unknown :-1 s=CLOSED pgs=736 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 >> 172.21.15.90:0/3548846257 conn(0x7fe28c076d40 msgr2=0x7fe28c0fde80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 shutdown_connections 2024-04-06T14:16:03.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.526+0000 7fe292814700 1 -- 172.21.15.90:0/3548846257 wait complete. 2024-04-06T14:16:03.541 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:03.541 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:16:03.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.725+0000 7f1ec029d700 1 -- 172.21.15.90:0/1150180901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb810f4b0 msgr2=0x7f1eb8107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.725+0000 7f1ec029d700 1 --2- 172.21.15.90:0/1150180901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8107db0 secure :-1 s=READY pgs=739 cs=0 l=1 rev1=1 crypto rx=0x7f1eb000b0a0 tx=0x7f1eb000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 -- 172.21.15.90:0/1150180901 shutdown_connections 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 --2- 172.21.15.90:0/1150180901 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb811a590 0x7f1eb811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 --2- 172.21.15.90:0/1150180901 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8107db0 unknown :-1 s=CLOSED pgs=739 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 --2- 172.21.15.90:0/1150180901 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f920 0x7f1eb81082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 -- 172.21.15.90:0/1150180901 >> 172.21.15.90:0/1150180901 conn(0x7f1eb806be00 msgr2=0x7f1eb806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 -- 172.21.15.90:0/1150180901 shutdown_connections 2024-04-06T14:16:03.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.726+0000 7f1ec029d700 1 -- 172.21.15.90:0/1150180901 wait complete. 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 Processor -- start 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 -- start start 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8117ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 0x7f1eb810fe40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb811a590 0x7f1eb8110380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb8110b50 con 0x7f1eb811a590 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb8110cc0 con 0x7f1eb810f920 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ec029d700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1eb81baea0 con 0x7f1eb810f4b0 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ebd838700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 0x7f1eb810fe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ebd838700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 0x7f1eb810fe40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:45100/0 (socket says 172.21.15.90:45100) 2024-04-06T14:16:03.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ebd838700 1 -- 172.21.15.90:0/1325518103 learned_addr learned my addr 172.21.15.90:0/1325518103 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.727+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8117ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb811a590 0x7f1eb8110380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 -- 172.21.15.90:0/1325518103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 msgr2=0x7f1eb810fe40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 0x7f1eb810fe40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 -- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb811a590 msgr2=0x7f1eb8110380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb811a590 0x7f1eb8110380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 -- 172.21.15.90:0/1325518103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1eb0009d00 con 0x7f1eb810f4b0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8117ea0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f1eb000b070 tx=0x7f1eb0007b60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1eaf7fe700 1 -- 172.21.15.90:0/1325518103 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb000e040 con 0x7f1eb810f4b0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1eaf7fe700 1 -- 172.21.15.90:0/1325518103 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1eb0003e10 con 0x7f1eb810f4b0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ec029d700 1 -- 172.21.15.90:0/1325518103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1eb81bb0f0 con 0x7f1eb810f4b0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1eaf7fe700 1 -- 172.21.15.90:0/1325518103 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb0012ac0 con 0x7f1eb810f4b0 2024-04-06T14:16:03.730 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.728+0000 7f1ec029d700 1 -- 172.21.15.90:0/1325518103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1eb81bb510 con 0x7f1eb810f4b0 2024-04-06T14:16:03.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.731+0000 7f1eaf7fe700 1 -- 172.21.15.90:0/1325518103 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1eb0012c20 con 0x7f1eb810f4b0 2024-04-06T14:16:03.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.732+0000 7f1eaf7fe700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 0x7f1ea40a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.732+0000 7f1eaf7fe700 1 -- 172.21.15.90:0/1325518103 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1eb00ce180 con 0x7f1eb810f4b0 2024-04-06T14:16:03.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 Processor -- start 2024-04-06T14:16:03.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 -- start start 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d930e100 0x5620d9318a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931b3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d9266990 con 0x5620d930e100 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d9264350 con 0x5620d930d7e0 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d925fc40 con 0x5620d9318fb0 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:45116/0 (socket says 172.21.15.90:45116) 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ebe039700 1 -- 172.21.15.90:0/3102451440 learned_addr learned my addr 172.21.15.90:0/3102451440 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.733+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931b3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 -- 172.21.15.90:0/3102451440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 msgr2=0x5620d931b3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931b3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 -- 172.21.15.90:0/3102451440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d930e100 msgr2=0x5620d9318a70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d930e100 0x5620d9318a70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 -- 172.21.15.90:0/3102451440 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5620d90e13b0 con 0x5620d930d7e0 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f1eb0009fd0 tx=0x7f1eb000b3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f752a61ac8e7a4a2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ec029d700 1 -- 172.21.15.90:0/3102451440 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb009bad0 con 0x5620d930d7e0 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ec029d700 1 -- 172.21.15.90:0/3102451440 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1eb009b3d0 con 0x5620d930d7e0 2024-04-06T14:16:03.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.734+0000 7f1ec029d700 1 -- 172.21.15.90:0/3102451440 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb0097ae0 con 0x5620d930d7e0 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3102451440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 msgr2=0x5620d930dbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f1eb0009fd0 tx=0x7f1eb000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3102451440 shutdown_connections 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931b3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d930dbc0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3102451440 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d930e100 0x5620d9318a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.735+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3102451440 >> 172.21.15.90:0/3102451440 conn(0x5620d926d6a0 msgr2=0x5620d9309c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.736+0000 7f1ebd838700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 0x7f1ea40a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.736+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3102451440 shutdown_connections 2024-04-06T14:16:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.736+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3102451440 wait complete. 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.736+0000 7f1ecc451b80 1 -- start start 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.736+0000 7f1ecc451b80 1 Processor -- start 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d93198d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9324410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d91fbd80 con 0x5620d9319e60 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d91fba20 con 0x5620d9318fb0 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d90e36a0 con 0x5620d930d7e0 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ecc451b80 10 client.0 _ll_register_callbacks cb 0x5620d9205470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54426/0 (socket says 172.21.15.90:54426) 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 -- 172.21.15.90:0/3396384912 learned_addr learned my addr 172.21.15.90:0/3396384912 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9324410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 -- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 msgr2=0x5620d93198d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebd838700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d93198d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d93198d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 -- 172.21.15.90:0/3396384912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 msgr2=0x5620d9324410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9324410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 -- 172.21.15.90:0/3396384912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1eb81bb510 con 0x5620d930d7e0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9324410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f1eb0097e40 tx=0x7f1eb00059d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=82f3c9521a7f7992 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ea1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb009f200 con 0x5620d930d7e0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.737+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1eb00a37c0 con 0x5620d930d7e0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.738+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5620d9220380 con 0x5620d930d7e0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.738+0000 7f1ebd838700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 0x7f1ea40a3750 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x5620d9305960 tx=0x7f1eb4009660 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.738+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1eb0019350 con 0x5620d930d7e0 2024-04-06T14:16:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.740+0000 7f1ecc451b80 10 client.26707 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.740+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5620d908db60 con 0x5620d930d7e0 2024-04-06T14:16:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1eb007e030 con 0x5620d930d7e0 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1eb009a9c0 con 0x5620d930d7e0 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ecc451b80 10 client.26707 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ecc451b80 10 client.26707 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ecc451b80 20 client.26707 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.741+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d9328710 0x5620d932ab30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.742+0000 7f1ecc451b80 4 client.26707 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.742+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5620d9258700 con 0x5620d9328710 2024-04-06T14:16:03.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.742+0000 7f1ebd838700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d9328710 0x5620d932ab30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.743+0000 7f1ea1ffb700 10 client.26707 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:03.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.743+0000 7f1ebd838700 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d9328710 0x5620d932ab30 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e67338539324c128 server_cookie=42a08555e6a29627 in_seq=0 out_seq=0 2024-04-06T14:16:03.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.744+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/3396384912 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5620d9258700 con 0x5620d9328710 2024-04-06T14:16:03.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.744+0000 7f1ea1ffb700 10 client.26707 handle_command_reply: tid=0 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.744+0000 7f1ecc451b80 1 client.26707 shutdown 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.744+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 msgr2=0x5620d9319390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.744+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f1eb0097e40 tx=0x7f1eb00059d0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 shutdown_connections 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9324410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d9328710 0x5620d932ab30 crc :-1 s=READY pgs=187 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d9319390 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3396384912 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d93198d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ea1ffb700 0 client.26707 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 >> 172.21.15.90:0/3396384912 conn(0x5620d926d6a0 msgr2=0x5620d930f640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 shutdown_connections 2024-04-06T14:16:03.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3396384912 wait complete. 2024-04-06T14:16:03.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.745+0000 7f1ecc451b80 20 client.26707 trim_cache size 0 max 16384 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 Processor -- start 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 -- start start 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d9324bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d931e910 con 0x5620d9319e60 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d9316760 con 0x5620d930d7e0 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d9319c40 con 0x5620d9318fb0 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60710/0 (socket says 172.21.15.90:60710) 2024-04-06T14:16:03.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 -- 172.21.15.90:0/3522784624 learned_addr learned my addr 172.21.15.90:0/3522784624 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe039700 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d9324bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebd838700 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 -- 172.21.15.90:0/3522784624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 msgr2=0x5620d931e3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931e3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 -- 172.21.15.90:0/3522784624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 msgr2=0x5620d9324bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d9324bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.763+0000 7f1ebe83a700 1 -- 172.21.15.90:0/3522784624 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5620d9225ea0 con 0x5620d9319e60 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.764+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f1ea800bbc0 tx=0x7f1ea800bed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=cc364b8ad05d593 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.764+0000 7f1eacda5700 1 -- 172.21.15.90:0/3522784624 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ea8017070 con 0x5620d9319e60 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.764+0000 7f1eacda5700 1 -- 172.21.15.90:0/3522784624 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ea8012e40 con 0x5620d9319e60 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.764+0000 7f1eacda5700 1 -- 172.21.15.90:0/3522784624 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ea80103d0 con 0x5620d9319e60 2024-04-06T14:16:03.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.765+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3522784624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 msgr2=0x5620d9313490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.765+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 secure :-1 s=READY pgs=740 cs=0 l=1 rev1=1 crypto rx=0x7f1ea800bbc0 tx=0x7f1ea800bed0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3522784624 shutdown_connections 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d9313490 unknown :-1 s=CLOSED pgs=740 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d930d7e0 0x5620d9324bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/3522784624 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d9318fb0 0x5620d931e3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3522784624 >> 172.21.15.90:0/3522784624 conn(0x5620d926d6a0 msgr2=0x5620d9312940 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3522784624 shutdown_connections 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.766+0000 7f1ecc451b80 1 -- 172.21.15.90:0/3522784624 wait complete. 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 -- start start 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 Processor -- start 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d931c660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d931cba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d931d260 con 0x5620d9319e60 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d931d3d0 con 0x5620d9318fb0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5620d937c510 con 0x5620d930d7e0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ecc451b80 10 client.0 _ll_register_callbacks cb 0x5620d9205470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ebd838700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d931c660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ebe039700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54432/0 (socket says 172.21.15.90:54432) 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ebe039700 1 -- 172.21.15.90:0/1715570807 learned_addr learned my addr 172.21.15.90:0/1715570807 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.767+0000 7f1ebe83a700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d931cba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 -- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 msgr2=0x5620d931c660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d931c660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 -- 172.21.15.90:0/1715570807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 msgr2=0x5620d931cba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:03.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d931cba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 -- 172.21.15.90:0/1715570807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ea800b900 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebd838700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d931c660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ebe039700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f1eb00a3290 tx=0x7f1eb009aad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9d5bbfd5efb262d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ea1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1eb00a3590 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1eb009b3d0 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1eb00a37c0 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1eb009f200 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ecc451b80 10 client.26713 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.768+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5620d90e5cd0 con 0x5620d930d7e0 2024-04-06T14:16:03.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1eb009b3d0 con 0x5620d930d7e0 2024-04-06T14:16:03.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1eb00a11a0 con 0x5620d930d7e0 2024-04-06T14:16:03.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 10 client.26713 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 10 client.26713 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 20 client.26713 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d937d6b0 0x5620d937faa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 4 client.26713 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.769+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5620d90aecf0 con 0x5620d937d6b0 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ebd838700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d937d6b0 0x5620d937faa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ea1ffb700 10 client.26713 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ebd838700 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d937d6b0 0x5620d937faa0 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=288b5b2587ed363a server_cookie=678c50a1bbd3a5a4 in_seq=0 out_seq=0 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ea1ffb700 1 -- 172.21.15.90:0/1715570807 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5620d90aecf0 con 0x5620d937d6b0 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ea1ffb700 10 client.26713 handle_command_reply: tid=0 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 client.26713 shutdown 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 msgr2=0x5620d931e180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f1eb00a3290 tx=0x7f1eb009aad0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 shutdown_connections 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5620d9319e60 0x5620d931cba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5620d937d6b0 0x5620d937faa0 crc :-1 s=READY pgs=188 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5620d930d7e0 0x5620d931e180 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ea1ffb700 0 client.26713 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 --2- 172.21.15.90:0/1715570807 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5620d9318fb0 0x5620d931c660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 >> 172.21.15.90:0/1715570807 conn(0x5620d926d6a0 msgr2=0x5620d9309690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 shutdown_connections 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.770+0000 7f1ecc451b80 1 -- 172.21.15.90:0/1715570807 wait complete. 2024-04-06T14:16:03.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ecc451b80 20 client.26713 trim_cache size 0 max 16384 2024-04-06T14:16:03.776 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:03.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 msgr2=0x7f1ea40a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 0x7f1ea40a3750 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x5620d9305960 tx=0x7f1eb4009660 comp rx=0 tx=0).stop 2024-04-06T14:16:03.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 msgr2=0x7f1eb8117ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8117ea0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f1eb000b070 tx=0x7f1eb0007b60 comp rx=0 tx=0).stop 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 shutdown_connections 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1ea40a1290 0x7f1ea40a3750 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1eb811a590 0x7f1eb8110380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1eb810f4b0 0x7f1eb8117ea0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 --2- 172.21.15.90:0/1325518103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1eb810f920 0x7f1eb810fe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 >> 172.21.15.90:0/1325518103 conn(0x7f1eb806be00 msgr2=0x7f1eb8118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 shutdown_connections 2024-04-06T14:16:03.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:03.771+0000 7f1ead5a6700 1 -- 172.21.15.90:0/1325518103 wait complete. 2024-04-06T14:16:04.788 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:16:04.788 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:16:04.788 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 -- 172.21.15.90:0/3509285593 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c00ff8e0 msgr2=0x7f89c00ffcc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 --2- 172.21.15.90:0/3509285593 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c00ff8e0 0x7f89c00ffcc0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f89b800c210 tx=0x7f89b800c520 comp rx=0 tx=0).stop 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 -- 172.21.15.90:0/3509285593 shutdown_connections 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 --2- 172.21.15.90:0/3509285593 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 0x7f89c010c2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 --2- 172.21.15.90:0/3509285593 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c00ff8e0 0x7f89c00ffcc0 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 --2- 172.21.15.90:0/3509285593 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c0100290 0x7f89c0104020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.973+0000 7f89c7c97700 1 -- 172.21.15.90:0/3509285593 >> 172.21.15.90:0/3509285593 conn(0x7f89c0074fd0 msgr2=0x7f89c00753e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.974+0000 7f89c7c97700 1 -- 172.21.15.90:0/3509285593 shutdown_connections 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.974+0000 7f89c7c97700 1 -- 172.21.15.90:0/3509285593 wait complete. 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.974+0000 7f89c7c97700 1 Processor -- start 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.974+0000 7f89c7c97700 1 -- start start 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.974+0000 7f89c7c97700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 0x7f89c01978f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c019bc90 0x7f89c019c110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89c019c710 con 0x7f89c00ff8e0 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89c019c880 con 0x7f89c0104750 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89c019ca40 con 0x7f89c019bc90 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60740/0 (socket says 172.21.15.90:60740) 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 -- 172.21.15.90:0/4045678778 learned_addr learned my addr 172.21.15.90:0/4045678778 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:04.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5232700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 0x7f89c01978f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c6234700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c019bc90 0x7f89c019c110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 -- 172.21.15.90:0/4045678778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c019bc90 msgr2=0x7f89c019c110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c019bc90 0x7f89c019c110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 -- 172.21.15.90:0/4045678778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 msgr2=0x7f89c01978f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 0x7f89c01978f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 -- 172.21.15.90:0/4045678778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89b800ae10 con 0x7f89c00ff8e0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c5a33700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f89b80082f0 tx=0x7f89b8008320 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89b6ffd700 1 -- 172.21.15.90:0/4045678778 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b800f040 con 0x7f89c00ff8e0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89b6ffd700 1 -- 172.21.15.90:0/4045678778 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f89b8009f00 con 0x7f89c00ff8e0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89b6ffd700 1 -- 172.21.15.90:0/4045678778 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b80058b0 con 0x7f89c00ff8e0 2024-04-06T14:16:04.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89c0194e90 con 0x7f89c00ff8e0 2024-04-06T14:16:04.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.975+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89c0195350 con 0x7f89c00ff8e0 2024-04-06T14:16:04.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.978+0000 7f89b6ffd700 1 -- 172.21.15.90:0/4045678778 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f89b800a070 con 0x7f89c00ff8e0 2024-04-06T14:16:04.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.978+0000 7f89b6ffd700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 0x7f89ac0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89b6ffd700 1 -- 172.21.15.90:0/4045678778 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f89b80ce270 con 0x7f89c00ff8e0 2024-04-06T14:16:04.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 Processor -- start 2024-04-06T14:16:04.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 -- start start 2024-04-06T14:16:04.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf8472cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84736920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf84684a10 con 0x55cf84736e60 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf846823d0 con 0x55cf8472d120 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.979+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf8467dcc0 con 0x55cf8472c800 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60750/0 (socket says 172.21.15.90:60750) 2024-04-06T14:16:04.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 -- 172.21.15.90:0/4180097792 learned_addr learned my addr 172.21.15.90:0/4180097792 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 -- 172.21.15.90:0/4180097792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 msgr2=0x55cf8472cbe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf8472cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 -- 172.21.15.90:0/4180097792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 msgr2=0x55cf84736920 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84736920 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 -- 172.21.15.90:0/4180097792 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf844ff430 con 0x55cf84736e60 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.980+0000 7f89c6234700 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7f89b0009cb0 tx=0x7f89b0011440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6e6896b26f5aa098 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.981+0000 7f89abfff700 1 -- 172.21.15.90:0/4180097792 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b0011e30 con 0x55cf84736e60 2024-04-06T14:16:04.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.981+0000 7f89abfff700 1 -- 172.21.15.90:0/4180097792 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f89b00044f0 con 0x55cf84736e60 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.981+0000 7f89abfff700 1 -- 172.21.15.90:0/4180097792 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b0019420 con 0x55cf84736e60 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89c5232700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 0x7f89ac0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/4180097792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 msgr2=0x55cf84739250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 secure :-1 s=READY pgs=742 cs=0 l=1 rev1=1 crypto rx=0x7f89b0009cb0 tx=0x7f89b0011440 comp rx=0 tx=0).stop 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/4180097792 shutdown_connections 2024-04-06T14:16:04.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84739250 unknown :-1 s=CLOSED pgs=742 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf8472cbe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/4180097792 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84736920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/4180097792 >> 172.21.15.90:0/4180097792 conn(0x55cf8468b720 msgr2=0x55cf84727b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/4180097792 shutdown_connections 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.982+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/4180097792 wait complete. 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.983+0000 7f89d3e4bb80 1 -- start start 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.983+0000 7f89c5232700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 0x7f89ac0a3750 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f89c0198900 tx=0x7f89bc00b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:04.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.983+0000 7f89d3e4bb80 1 Processor -- start 2024-04-06T14:16:04.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.983+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.983+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84734390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf84619e00 con 0x55cf84736e60 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf84619aa0 con 0x55cf8472d120 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf84501720 con 0x55cf8472c800 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89d3e4bb80 10 client.0 _ll_register_callbacks cb 0x55cf846234f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5a33700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:45160/0 (socket says 172.21.15.90:45160) 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5a33700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54478/0 (socket says 172.21.15.90:54478) 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 -- 172.21.15.90:0/1950958182 learned_addr learned my addr 172.21.15.90:0/1950958182 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c6234700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84734390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 -- 172.21.15.90:0/1950958182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 msgr2=0x55cf847368d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:04.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 -- 172.21.15.90:0/1950958182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 msgr2=0x55cf84734390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84734390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 -- 172.21.15.90:0/1950958182 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89b0009960 con 0x55cf8472d120 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c6234700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84734390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5a33700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89c5232700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f89bc010ec0 tx=0x7f89bc0047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5ee3bd0cf793b4b0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89a97fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.984+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89bc01e620 con 0x55cf8472d120 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f89bc017070 con 0x55cf8472d120 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cf8463e400 con 0x55cf8472d120 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f89bc01e780 con 0x55cf8472d120 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 10 client.26948 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:04.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cf844abbe0 con 0x55cf8472d120 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55cf8463e400 con 0x55cf8472d120 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f89bc026960 con 0x55cf8472d120 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 10 client.26948 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 10 client.26948 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 20 client.26948 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf8474ea90 0x55cf84750eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 4 client.26948 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.985+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cf84676780 con 0x55cf8474ea90 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.986+0000 7f89c5a33700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf8474ea90 0x55cf84750eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.986+0000 7f89a97fa700 10 client.26948 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:04.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.986+0000 7f89c5a33700 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf8474ea90 0x55cf84750eb0 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=502e1dab4684aeeb server_cookie=d974f24333ca018f in_seq=0 out_seq=0 2024-04-06T14:16:04.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89a97fa700 1 -- 172.21.15.90:0/1950958182 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55cf84676780 con 0x55cf8474ea90 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89a97fa700 10 client.26948 handle_command_reply: tid=0 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 client.26948 shutdown 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 msgr2=0x55cf84733c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f89bc010ec0 tx=0x7f89bc0047c0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 shutdown_connections 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf84734390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf8474ea90 0x55cf84750eb0 crc :-1 s=READY pgs=189 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472c800 0x55cf847368d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/1950958182 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472d120 0x55cf84733c50 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 >> 172.21.15.90:0/1950958182 conn(0x55cf8468b720 msgr2=0x55cf84726ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89a97fa700 0 client.26948 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 shutdown_connections 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.987+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/1950958182 wait complete. 2024-04-06T14:16:04.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:04.988+0000 7f89d3e4bb80 20 client.26948 trim_cache size 0 max 16384 2024-04-06T14:16:05.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.007+0000 7f89d3e4bb80 1 Processor -- start 2024-04-06T14:16:05.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.007+0000 7f89d3e4bb80 1 -- start start 2024-04-06T14:16:05.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf8472c800 0x55cf84733510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84755820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf8463e400 con 0x55cf8472c800 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf8474c190 con 0x55cf84736e60 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf84728840 con 0x55cf8472d120 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89c6234700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89c6234700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:45164/0 (socket says 172.21.15.90:45164) 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89c6234700 1 -- 172.21.15.90:0/3075905917 learned_addr learned my addr 172.21.15.90:0/3075905917 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89c5232700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84755820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.008+0000 7f89c5a33700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf8472c800 0x55cf84733510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 -- 172.21.15.90:0/3075905917 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 msgr2=0x55cf84755820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84755820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 -- 172.21.15.90:0/3075905917 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf8472c800 msgr2=0x55cf84733510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf8472c800 0x55cf84733510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 -- 172.21.15.90:0/3075905917 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf84733d80 con 0x55cf84736e60 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c6234700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f89b0009c50 tx=0x7f89b0011cb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a1eeb98f77f883a0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89c5232700 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84755820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:16:05.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89b4da5700 1 -- 172.21.15.90:0/3075905917 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b0004200 con 0x55cf84736e60 2024-04-06T14:16:05.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89b4da5700 1 -- 172.21.15.90:0/3075905917 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f89b0004360 con 0x55cf84736e60 2024-04-06T14:16:05.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.009+0000 7f89b4da5700 1 -- 172.21.15.90:0/3075905917 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b0003cb0 con 0x55cf84736e60 2024-04-06T14:16:05.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/3075905917 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 msgr2=0x55cf847275a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f89b0009c50 tx=0x7f89b0011cb0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/3075905917 shutdown_connections 2024-04-06T14:16:05.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf84736e60 0x55cf847275a0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf8472c800 0x55cf84733510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/3075905917 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84755820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/3075905917 >> 172.21.15.90:0/3075905917 conn(0x55cf8468b720 msgr2=0x55cf8472ef20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/3075905917 shutdown_connections 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.010+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/3075905917 wait complete. 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 -- start start 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 Processor -- start 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472c800 0x55cf84754630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84754b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf847b69d0 con 0x55cf84736e60 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf847b6b40 con 0x55cf8472c800 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf847b6d00 con 0x55cf8472d120 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89d3e4bb80 10 client.0 _ll_register_callbacks cb 0x55cf846234f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:05.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60784/0 (socket says 172.21.15.90:60784) 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 -- 172.21.15.90:0/931949597 learned_addr learned my addr 172.21.15.90:0/931949597 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c5a33700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472c800 0x55cf84754630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c5232700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84754b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 -- 172.21.15.90:0/931949597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 msgr2=0x55cf84754b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84754b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 -- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472c800 msgr2=0x55cf84754630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472c800 0x55cf84754630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.011+0000 7f89c6234700 1 -- 172.21.15.90:0/931949597 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55cf844ff430 con 0x55cf84736e60 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.012+0000 7f89c6234700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f89b001b040 tx=0x7f89b0003850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d6f0d67d5f677e8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.012+0000 7f89a97fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.012+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89b0004a70 con 0x55cf84736e60 2024-04-06T14:16:05.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.012+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f89b0004bd0 con 0x55cf84736e60 2024-04-06T14:16:05.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.013+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f89b00044f0 con 0x55cf84736e60 2024-04-06T14:16:05.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.013+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f89b00056d0 con 0x55cf84736e60 2024-04-06T14:16:05.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.013+0000 7f89d3e4bb80 10 client.17250 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:05.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.013+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55cf84723be0 con 0x55cf84736e60 2024-04-06T14:16:05.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.014+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f89b00044f0 con 0x55cf84736e60 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.014+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f89b0009ce0 con 0x55cf84736e60 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 10 client.17250 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 10 client.17250 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 20 client.17250 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf847be000 0x55cf847c03f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 4 client.17250 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55cf844ccd70 con 0x55cf847be000 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89c5a33700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf847be000 0x55cf847c03f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89a97fa700 10 client.17250 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.015+0000 7f89c5a33700 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf847be000 0x55cf847c03f0 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1e2971e2ed427535 server_cookie=ad0d55fed8df1769 in_seq=0 out_seq=0 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89a97fa700 1 -- 172.21.15.90:0/931949597 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55cf844ccd70 con 0x55cf847be000 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89a97fa700 10 client.17250 handle_command_reply: tid=0 2024-04-06T14:16:05.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 client.17250 shutdown 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 msgr2=0x55cf847b6490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 secure :-1 s=READY pgs=743 cs=0 l=1 rev1=1 crypto rx=0x7f89b001b040 tx=0x7f89b0003850 comp rx=0 tx=0).stop 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 shutdown_connections 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55cf84736e60 0x55cf847b6490 unknown :-1 s=CLOSED pgs=743 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55cf8472c800 0x55cf84754630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55cf847be000 0x55cf847c03f0 crc :-1 s=READY pgs=190 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 --2- 172.21.15.90:0/931949597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55cf8472d120 0x55cf84754b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89a97fa700 0 client.17250 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.016+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 >> 172.21.15.90:0/931949597 conn(0x55cf8468b720 msgr2=0x55cf847274b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 shutdown_connections 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89d3e4bb80 1 -- 172.21.15.90:0/931949597 wait complete. 2024-04-06T14:16:05.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89d3e4bb80 20 client.17250 trim_cache size 0 max 16384 2024-04-06T14:16:05.021 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:05.021 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:16:05.021 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "b8cddda4-09ca-4972-a66f-8ecbf3f710dd", 2024-04-06T14:16:05.021 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:16:05.021 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 msgr2=0x7f89ac0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 0x7f89ac0a3750 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f89c0198900 tx=0x7f89bc00b410 comp rx=0 tx=0).stop 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 msgr2=0x7f89c01973b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 secure :-1 s=READY pgs=741 cs=0 l=1 rev1=1 crypto rx=0x7f89b80082f0 tx=0x7f89b8008320 comp rx=0 tx=0).stop 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 shutdown_connections 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f89ac0a1290 0x7f89ac0a3750 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f89c019bc90 0x7f89c019c110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f89c00ff8e0 0x7f89c01973b0 unknown :-1 s=CLOSED pgs=741 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 --2- 172.21.15.90:0/4045678778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f89c0104750 0x7f89c01978f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 >> 172.21.15.90:0/4045678778 conn(0x7f89c0074fd0 msgr2=0x7f89c00fd950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 shutdown_connections 2024-04-06T14:16:05.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.017+0000 7f89c7c97700 1 -- 172.21.15.90:0/4045678778 wait complete. 2024-04-06T14:16:05.031 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:16:05.031 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:16:05.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.190+0000 7f0606c94700 1 -- 172.21.15.90:0/24744597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0600100030 msgr2=0x7f060010c240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.190+0000 7f0606c94700 1 --2- 172.21.15.90:0/24744597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0600100030 0x7f060010c240 secure :-1 s=READY pgs=744 cs=0 l=1 rev1=1 crypto rx=0x7f05f800b0d0 tx=0x7f05f800b3e0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- 172.21.15.90:0/24744597 shutdown_connections 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- 172.21.15.90:0/24744597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0600100030 0x7f060010c240 unknown :-1 s=CLOSED pgs=744 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- 172.21.15.90:0/24744597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 0x7f0600104790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- 172.21.15.90:0/24744597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06000ff670 0x7f06000ffaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- 172.21.15.90:0/24744597 >> 172.21.15.90:0/24744597 conn(0x7f0600074f00 msgr2=0x7f0600075310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- 172.21.15.90:0/24744597 shutdown_connections 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- 172.21.15.90:0/24744597 wait complete. 2024-04-06T14:16:05.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 Processor -- start 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- start start 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0600100030 0x7f0600196780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 0x7f060019ab20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f060019b160 con 0x7f06000ff670 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f060019b2d0 con 0x7f06001043b0 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0606c94700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f060019b490 con 0x7f0600100030 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.191+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60798/0 (socket says 172.21.15.90:60798) 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 -- 172.21.15.90:0/3602533984 learned_addr learned my addr 172.21.15.90:0/3602533984 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0605231700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 0x7f060019ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.195 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 -- 172.21.15.90:0/3602533984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0600100030 msgr2=0x7f0600196780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0600100030 0x7f0600196780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 -- 172.21.15.90:0/3602533984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 msgr2=0x7f060019ab20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 0x7f060019ab20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 -- 172.21.15.90:0/3602533984 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05f8009d00 con 0x7f06000ff670 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0604a30700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f05f400b500 tx=0x7f05f400b8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f05fdffb700 1 -- 172.21.15.90:0/3602533984 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f4011830 con 0x7f06000ff670 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f05fdffb700 1 -- 172.21.15.90:0/3602533984 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05f4011e50 con 0x7f06000ff670 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f060019b770 con 0x7f06000ff670 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f05fdffb700 1 -- 172.21.15.90:0/3602533984 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f400f2e0 con 0x7f06000ff670 2024-04-06T14:16:05.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.192+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06001a2fb0 con 0x7f06000ff670 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.194+0000 7f05fdffb700 1 -- 172.21.15.90:0/3602533984 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f05f4010580 con 0x7f06000ff670 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.195+0000 7f05fdffb700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 0x7f05f00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.195+0000 7f05fffff700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 0x7f05f00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.195+0000 7f05fdffb700 1 -- 172.21.15.90:0/3602533984 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f05f40cc6e0 con 0x7f06000ff670 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.195+0000 7f05fffff700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 0x7f05f00a3750 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f05ec005950 tx=0x7f05ec0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 Processor -- start 2024-04-06T14:16:05.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 -- start start 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683ba10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de56836ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5678e990 con 0x55de568371e0 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5678c350 con 0x55de56836820 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0612e48b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de56787c40 con 0x55de5683b630 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.196+0000 7f0605231700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60812/0 (socket says 172.21.15.90:60812) 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 -- 172.21.15.90:0/3721920359 learned_addr learned my addr 172.21.15.90:0/3721920359 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0604a30700 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f05fffff700 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de56836ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 -- 172.21.15.90:0/3721920359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 msgr2=0x55de5683ba10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683ba10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 -- 172.21.15.90:0/3721920359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 msgr2=0x55de56836ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de56836ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 -- 172.21.15.90:0/3721920359 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55de566093b0 con 0x55de568371e0 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f0605231700 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f05f8009390 tx=0x7f05f80093c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=46a29e7222177201 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f05eaffd700 1 -- 172.21.15.90:0/3721920359 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f800e040 con 0x55de568371e0 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f05eaffd700 1 -- 172.21.15.90:0/3721920359 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05f8009b80 con 0x55de568371e0 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.197+0000 7f05eaffd700 1 -- 172.21.15.90:0/3721920359 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f80077d0 con 0x55de568371e0 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- 172.21.15.90:0/3721920359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 msgr2=0x55de568433b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 secure :-1 s=READY pgs=746 cs=0 l=1 rev1=1 crypto rx=0x7f05f8009390 tx=0x7f05f80093c0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- 172.21.15.90:0/3721920359 shutdown_connections 2024-04-06T14:16:05.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de568433b0 unknown :-1 s=CLOSED pgs=746 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 --2- 172.21.15.90:0/3721920359 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de56836ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- 172.21.15.90:0/3721920359 >> 172.21.15.90:0/3721920359 conn(0x55de567956a0 msgr2=0x55de56831980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- 172.21.15.90:0/3721920359 shutdown_connections 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- 172.21.15.90:0/3721920359 wait complete. 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 -- start start 2024-04-06T14:16:05.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.198+0000 7f0612e48b80 1 Processor -- start 2024-04-06T14:16:05.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de568371e0 0x55de568400d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de5683b630 0x55de5685d610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de56723d80 con 0x55de56836820 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de56723a20 con 0x55de5683b630 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5660b6a0 con 0x55de568371e0 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.199+0000 7f0612e48b80 10 client.0 _ll_register_callbacks cb 0x55de5672d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60826/0 (socket says 172.21.15.90:60826) 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 -- 172.21.15.90:0/4202117797 learned_addr learned my addr 172.21.15.90:0/4202117797 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f05fffff700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de568371e0 0x55de568400d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 -- 172.21.15.90:0/4202117797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de568371e0 msgr2=0x55de568400d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de568371e0 0x55de568400d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 -- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de5683b630 msgr2=0x55de5685d610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de5683b630 0x55de5685d610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 -- 172.21.15.90:0/4202117797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f060010cb40 con 0x55de56836820 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0604a30700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f05f400b4d0 tx=0x7f05f4010400 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7b0bb7d4107665bd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f05dffff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f40b1070 con 0x55de56836820 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55de56780700 con 0x55de56836820 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.200+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f05f40174d0 con 0x55de56836820 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f05f40b2260 con 0x55de56836820 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 10 client.17262 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:05.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55de565b5b60 con 0x55de56836820 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05f4063020 con 0x55de56836820 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f05f4098dd0 con 0x55de56836820 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 10 client.17262 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 10 client.17262 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 20 client.17262 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568728f0 0x55de56874d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 4 client.17262 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.201+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55de565d6cf0 con 0x55de568728f0 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.203+0000 7f05fffff700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568728f0 0x55de56874d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.203+0000 7f05fffff700 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568728f0 0x55de56874d10 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=17543d2bd9a78c4d server_cookie=84ff3fca0f6f225b in_seq=0 out_seq=0 2024-04-06T14:16:05.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.203+0000 7f05dffff700 10 client.17262 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.204+0000 7f05dffff700 1 -- 172.21.15.90:0/4202117797 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55de565d6cf0 con 0x55de568728f0 2024-04-06T14:16:05.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.204+0000 7f05dffff700 10 client.17262 handle_command_reply: tid=0 2024-04-06T14:16:05.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.204+0000 7f0612e48b80 1 client.17262 shutdown 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.204+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 msgr2=0x55de5683fb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.204+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 secure :-1 s=READY pgs=747 cs=0 l=1 rev1=1 crypto rx=0x7f05f400b4d0 tx=0x7f05f4010400 comp rx=0 tx=0).stop 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 shutdown_connections 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de5683b630 0x55de5685d610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de5683fb90 unknown :-1 s=CLOSED pgs=747 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568728f0 0x55de56874d10 crc :-1 s=READY pgs=191 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 --2- 172.21.15.90:0/4202117797 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de568371e0 0x55de568400d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 >> 172.21.15.90:0/4202117797 conn(0x55de567956a0 msgr2=0x55de56830390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f05dffff700 0 client.17262 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 shutdown_connections 2024-04-06T14:16:05.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.205+0000 7f0612e48b80 1 -- 172.21.15.90:0/4202117797 wait complete. 2024-04-06T14:16:05.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.206+0000 7f0612e48b80 20 client.17262 trim_cache size 0 max 16384 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 Processor -- start 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 -- start start 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de568371e0 0x55de5683d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683d700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de56859b20 con 0x55de56836820 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5683e100 con 0x55de568371e0 2024-04-06T14:16:05.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0612e48b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de56840400 con 0x55de5683b630 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60832/0 (socket says 172.21.15.90:60832) 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 -- 172.21.15.90:0/1892463407 learned_addr learned my addr 172.21.15.90:0/1892463407 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0605231700 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683d700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f05fffff700 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de568371e0 0x55de5683d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 -- 172.21.15.90:0/1892463407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 msgr2=0x55de5683d700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683d700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 -- 172.21.15.90:0/1892463407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de568371e0 msgr2=0x55de5683d1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de568371e0 0x55de5683d1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.228+0000 7f0604a30700 1 -- 172.21.15.90:0/1892463407 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55de5674dea0 con 0x55de56836820 2024-04-06T14:16:05.231 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.229+0000 7f0604a30700 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f05f4096350 tx=0x7f05f4099a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3bac6fa67cfb3a23 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.229+0000 7f05eb7fe700 1 -- 172.21.15.90:0/1892463407 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f40b1070 con 0x55de56836820 2024-04-06T14:16:05.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.229+0000 7f05eb7fe700 1 -- 172.21.15.90:0/1892463407 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05f4063030 con 0x55de56836820 2024-04-06T14:16:05.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.229+0000 7f05eb7fe700 1 -- 172.21.15.90:0/1892463407 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f40a1ba0 con 0x55de56836820 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 -- 172.21.15.90:0/1892463407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 msgr2=0x55de568595e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 secure :-1 s=READY pgs=748 cs=0 l=1 rev1=1 crypto rx=0x7f05f4096350 tx=0x7f05f4099a60 comp rx=0 tx=0).stop 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 -- 172.21.15.90:0/1892463407 shutdown_connections 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5683d700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de56836820 0x55de568595e0 unknown :-1 s=CLOSED pgs=748 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1892463407 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de568371e0 0x55de5683d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.230+0000 7f0612e48b80 1 -- 172.21.15.90:0/1892463407 >> 172.21.15.90:0/1892463407 conn(0x55de567956a0 msgr2=0x55de5683a070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.231+0000 7f0612e48b80 1 -- 172.21.15.90:0/1892463407 shutdown_connections 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.231+0000 7f0612e48b80 1 -- 172.21.15.90:0/1892463407 wait complete. 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.231+0000 7f0612e48b80 1 -- start start 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.231+0000 7f0612e48b80 1 Processor -- start 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de5685d400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de5685bd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5685c7b0 con 0x55de568371e0 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5685c920 con 0x55de56836820 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55de5685cae0 con 0x55de5683b630 2024-04-06T14:16:05.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0612e48b80 10 client.0 _ll_register_callbacks cb 0x55de5672d470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54560/0 (socket says 172.21.15.90:54560) 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 -- 172.21.15.90:0/1739613775 learned_addr learned my addr 172.21.15.90:0/1739613775 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 -- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 msgr2=0x55de5685d400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de5685d400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 -- 172.21.15.90:0/1739613775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 msgr2=0x55de5685bd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de5685bd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.232+0000 7f0605231700 1 -- 172.21.15.90:0/1739613775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55de565b5b60 con 0x55de5683b630 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f0605231700 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f05f8009390 tx=0x7f05f800bd80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e7be3998d4796f6d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f05dffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f05f800e050 con 0x55de5683b630 2024-04-06T14:16:05.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f05f8004c60 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55de56748380 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f05f800b770 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f0612e48b80 10 client.26743 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55de5660dcd0 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.233+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55de56748380 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f05f8019030 con 0x55de5683b630 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 10 client.26743 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 10 client.26743 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 20 client.26743 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568c7830 0x55de568c9c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:05.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 4 client.26743 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:16:05.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55de5660bce0 con 0x55de568c7830 2024-04-06T14:16:05.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0604a30700 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568c7830 0x55de568c9c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f0604a30700 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568c7830 0x55de568c9c50 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7e02a3d24699dbd9 server_cookie=b9fef91b3d30222 in_seq=0 out_seq=0 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.234+0000 7f05dffff700 10 client.26743 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f05dffff700 1 -- 172.21.15.90:0/1739613775 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x55de5660bce0 con 0x55de568c7830 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f05dffff700 10 client.26743 handle_command_reply: tid=0 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 client.26743 shutdown 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 msgr2=0x55de5685c270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f05f8009390 tx=0x7f05f800bd80 comp rx=0 tx=0).stop 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 shutdown_connections 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55de568c7830 0x55de568c9c50 crc :-1 s=READY pgs=192 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55de5683b630 0x55de5685c270 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55de56836820 0x55de5685d400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 --2- 172.21.15.90:0/1739613775 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55de568371e0 0x55de5685bd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 >> 172.21.15.90:0/1739613775 conn(0x55de567956a0 msgr2=0x55de5683e3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f05dffff700 0 client.26743 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (131 inodes in the stack)", 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:16:05.241 INFO:teuthology.orchestra.run.smithi090.stdout: "b8cddda4-09ca-4972-a66f-8ecbf3f710dd": { 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "b8cddda4-09ca-4972-a66f-8ecbf3f710dd", 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:16:05.242 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 shutdown_connections 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 1 -- 172.21.15.90:0/1739613775 wait complete. 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.235+0000 7f0612e48b80 20 client.26743 trim_cache size 0 max 16384 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 msgr2=0x7f05f00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 0x7f05f00a3750 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f05ec005950 tx=0x7f05ec0058e0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 msgr2=0x7f0600196240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 secure :-1 s=READY pgs=745 cs=0 l=1 rev1=1 crypto rx=0x7f05f400b500 tx=0x7f05f400b8c0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 shutdown_connections 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f05f00a1290 0x7f05f00a3750 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06001043b0 0x7f060019ab20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06000ff670 0x7f0600196240 unknown :-1 s=CLOSED pgs=745 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 --2- 172.21.15.90:0/3602533984 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0600100030 0x7f0600196780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 >> 172.21.15.90:0/3602533984 conn(0x7f0600074f00 msgr2=0x7f0600102fe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 shutdown_connections 2024-04-06T14:16:05.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:05.236+0000 7f0606c94700 1 -- 172.21.15.90:0/3602533984 wait complete. 2024-04-06T14:16:05.250 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:05 smithi090 ceph-mon[43420]: pgmap v331: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 9.3 MiB/s wr, 716 op/s 2024-04-06T14:16:05.251 INFO:tasks.cephfs.filesystem:scrub status for tag:b8cddda4-09ca-4972-a66f-8ecbf3f710dd - {'path': '/', 'tag': 'b8cddda4-09ca-4972-a66f-8ecbf3f710dd', 'options': 'recursive,force'} 2024-04-06T14:16:05.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:05 smithi134 ceph-mon[47549]: pgmap v331: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 9.3 MiB/s wr, 716 op/s 2024-04-06T14:16:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:05 smithi105 ceph-mon[46969]: pgmap v331: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 9.3 MiB/s wr, 716 op/s 2024-04-06T14:16:06.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:06 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:16:06.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:06 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:06 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:16:06.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:06 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:06 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:16:06.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:06 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:16:06.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:06 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:06 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:16:06.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:06 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:06 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:16:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:06 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:16:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:06 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:06 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:16:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:06 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:06.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:06 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:16:07.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:07 smithi134 ceph-mon[47549]: pgmap v332: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 13 MiB/s wr, 1.03k op/s 2024-04-06T14:16:07.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:07 smithi134 ceph-mon[47549]: osdmap e93: 12 total, 12 up, 12 in 2024-04-06T14:16:07.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:07 smithi105 ceph-mon[46969]: pgmap v332: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 13 MiB/s wr, 1.03k op/s 2024-04-06T14:16:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:07 smithi105 ceph-mon[46969]: osdmap e93: 12 total, 12 up, 12 in 2024-04-06T14:16:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:07 smithi090 ceph-mon[43420]: pgmap v332: 97 pgs: 97 active+clean; 7.0 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 13 MiB/s wr, 1.03k op/s 2024-04-06T14:16:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:07 smithi090 ceph-mon[43420]: osdmap e93: 12 total, 12 up, 12 in 2024-04-06T14:16:08.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 620.0 s, 1157.8 tps, lat 99.267 ms stddev 188.921 2024-04-06T14:16:09.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:08 smithi134 ceph-mon[47549]: pgmap v334: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 22 MiB/s wr, 1.84k op/s 2024-04-06T14:16:09.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:08 smithi105 ceph-mon[46969]: pgmap v334: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 22 MiB/s wr, 1.84k op/s 2024-04-06T14:16:09.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:08 smithi090 ceph-mon[43420]: pgmap v334: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.9 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 22 MiB/s wr, 1.84k op/s 2024-04-06T14:16:11.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:11 smithi134 ceph-mon[47549]: pgmap v335: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.8 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 23 MiB/s wr, 1.95k op/s 2024-04-06T14:16:11.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:11 smithi105 ceph-mon[46969]: pgmap v335: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.8 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 23 MiB/s wr, 1.95k op/s 2024-04-06T14:16:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:11 smithi090 ceph-mon[43420]: pgmap v335: 97 pgs: 11 active+clean+snaptrim, 22 active+clean+snaptrim_wait, 64 active+clean; 6.8 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 23 MiB/s wr, 1.95k op/s 2024-04-06T14:16:12.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:12 smithi134 ceph-mon[47549]: pgmap v336: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 22 MiB/s wr, 1.88k op/s 2024-04-06T14:16:12.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:12 smithi105 ceph-mon[46969]: pgmap v336: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 22 MiB/s wr, 1.88k op/s 2024-04-06T14:16:12.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:12 smithi090 ceph-mon[43420]: pgmap v336: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 102 KiB/s rd, 22 MiB/s wr, 1.88k op/s 2024-04-06T14:16:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 625.0 s, 1777.8 tps, lat 55.804 ms stddev 66.070 2024-04-06T14:16:14.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:13 smithi134 ceph-mon[47549]: osdmap e94: 12 total, 12 up, 12 in 2024-04-06T14:16:14.257 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:13 smithi105 ceph-mon[46969]: osdmap e94: 12 total, 12 up, 12 in 2024-04-06T14:16:14.293 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:13 smithi090 ceph-mon[43420]: osdmap e94: 12 total, 12 up, 12 in 2024-04-06T14:16:15.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:14 smithi105 ceph-mon[46969]: pgmap v338: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 112 KiB/s rd, 20 MiB/s wr, 1.71k op/s 2024-04-06T14:16:15.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:14 smithi090 ceph-mon[43420]: pgmap v338: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 112 KiB/s rd, 20 MiB/s wr, 1.71k op/s 2024-04-06T14:16:15.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:14 smithi134 ceph-mon[47549]: pgmap v338: 97 pgs: 97 active+clean; 6.1 GiB data, 17 GiB used, 1.0 TiB / 1.0 TiB avail; 112 KiB/s rd, 20 MiB/s wr, 1.71k op/s 2024-04-06T14:16:17.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:16 smithi105 ceph-mon[46969]: pgmap v339: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:17.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:16 smithi090 ceph-mon[43420]: pgmap v339: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:17.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:16 smithi134 ceph-mon[47549]: pgmap v339: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:18.030 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 630.0 s, 965.5 tps, lat 104.334 ms stddev 126.109 2024-04-06T14:16:19.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:18 smithi134 ceph-mon[47549]: pgmap v340: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 11 MiB/s wr, 834 op/s 2024-04-06T14:16:19.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:18 smithi105 ceph-mon[46969]: pgmap v340: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 11 MiB/s wr, 834 op/s 2024-04-06T14:16:19.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:18 smithi090 ceph-mon[43420]: pgmap v340: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 11 MiB/s wr, 834 op/s 2024-04-06T14:16:21.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:20 smithi105 ceph-mon[46969]: pgmap v341: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 12 MiB/s wr, 798 op/s 2024-04-06T14:16:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:20 smithi090 ceph-mon[43420]: pgmap v341: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 12 MiB/s wr, 798 op/s 2024-04-06T14:16:21.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:20 smithi134 ceph-mon[47549]: pgmap v341: 97 pgs: 97 active+clean; 6.1 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 12 MiB/s wr, 798 op/s 2024-04-06T14:16:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 635.0 s, 1348.2 tps, lat 73.986 ms stddev 81.980 2024-04-06T14:16:23.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:22 smithi105 ceph-mon[46969]: pgmap v342: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 15 MiB/s wr, 859 op/s 2024-04-06T14:16:23.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:22 smithi090 ceph-mon[43420]: pgmap v342: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 15 MiB/s wr, 859 op/s 2024-04-06T14:16:23.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:22 smithi134 ceph-mon[47549]: pgmap v342: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 15 MiB/s wr, 859 op/s 2024-04-06T14:16:25.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:24 smithi105 ceph-mon[46969]: pgmap v343: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 14 MiB/s wr, 815 op/s 2024-04-06T14:16:25.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:24 smithi090 ceph-mon[43420]: pgmap v343: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 14 MiB/s wr, 815 op/s 2024-04-06T14:16:25.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:24 smithi134 ceph-mon[47549]: pgmap v343: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 14 MiB/s wr, 815 op/s 2024-04-06T14:16:27.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:26 smithi090 ceph-mon[43420]: pgmap v344: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 779 op/s 2024-04-06T14:16:27.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:26 smithi134 ceph-mon[47549]: pgmap v344: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 779 op/s 2024-04-06T14:16:27.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:26 smithi105 ceph-mon[46969]: pgmap v344: 97 pgs: 97 active+clean; 6.2 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 779 op/s 2024-04-06T14:16:28.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 640.0 s, 1637.8 tps, lat 61.048 ms stddev 75.431 2024-04-06T14:16:29.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:28 smithi134 ceph-mon[47549]: pgmap v345: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 844 op/s 2024-04-06T14:16:29.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:28 smithi105 ceph-mon[46969]: pgmap v345: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 844 op/s 2024-04-06T14:16:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:28 smithi090 ceph-mon[43420]: pgmap v345: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 16 MiB/s wr, 844 op/s 2024-04-06T14:16:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:31 smithi090 ceph-mon[43420]: pgmap v346: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 645 op/s 2024-04-06T14:16:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:31 smithi134 ceph-mon[47549]: pgmap v346: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 645 op/s 2024-04-06T14:16:31.500 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:31 smithi105 ceph-mon[46969]: pgmap v346: 97 pgs: 97 active+clean; 6.3 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 14 MiB/s wr, 645 op/s 2024-04-06T14:16:33.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 645.0 s, 1235.4 tps, lat 79.284 ms stddev 127.357 2024-04-06T14:16:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:33 smithi134 ceph-mon[47549]: pgmap v347: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 25 MiB/s wr, 1.60k op/s 2024-04-06T14:16:33.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:33 smithi105 ceph-mon[46969]: pgmap v347: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 25 MiB/s wr, 1.60k op/s 2024-04-06T14:16:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:33 smithi090 ceph-mon[43420]: pgmap v347: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 25 MiB/s wr, 1.60k op/s 2024-04-06T14:16:35.252 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:16:35.252 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:16:35.299 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:35 smithi090 ceph-mon[43420]: pgmap v348: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 22 MiB/s wr, 1.53k op/s 2024-04-06T14:16:35.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:35 smithi134 ceph-mon[47549]: pgmap v348: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 22 MiB/s wr, 1.53k op/s 2024-04-06T14:16:35.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.448+0000 7fc0fee29700 1 -- 172.21.15.90:0/1110985938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8069580 msgr2=0x7fc0f8069960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.448+0000 7fc0fee29700 1 --2- 172.21.15.90:0/1110985938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8069580 0x7fc0f8069960 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7fc0ec009990 tx=0x7fc0ec009ca0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 -- 172.21.15.90:0/1110985938 shutdown_connections 2024-04-06T14:16:35.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 --2- 172.21.15.90:0/1110985938 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8106090 0x7fc0f810dc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 --2- 172.21.15.90:0/1110985938 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8069580 0x7fc0f8069960 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 --2- 172.21.15.90:0/1110985938 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8069f30 0x7fc0f8105960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 -- 172.21.15.90:0/1110985938 >> 172.21.15.90:0/1110985938 conn(0x7fc0f8076db0 msgr2=0x7fc0f80771c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 -- 172.21.15.90:0/1110985938 shutdown_connections 2024-04-06T14:16:35.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.449+0000 7fc0fee29700 1 -- 172.21.15.90:0/1110985938 wait complete. 2024-04-06T14:16:35.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 Processor -- start 2024-04-06T14:16:35.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 -- start start 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 0x7fc0f8110320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 0x7fc0f8110860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0f81156b0 con 0x7fc0f8069580 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0f8115820 con 0x7fc0f8114c00 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fee29700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0f81159e0 con 0x7fc0f8106090 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fd3c6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0f7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 0x7fc0f8110860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fd3c6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55196/0 (socket says 172.21.15.90:55196) 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fd3c6700 1 -- 172.21.15.90:0/2148598092 learned_addr learned my addr 172.21.15.90:0/2148598092 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.450+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 0x7fc0f8110320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 -- 172.21.15.90:0/2148598092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 msgr2=0x7fc0f8110860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 0x7fc0f8110860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 -- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 msgr2=0x7fc0f8110320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 0x7fc0f8110320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 -- 172.21.15.90:0/2148598092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0ec0095f0 con 0x7fc0f8114c00 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 0x7fc0f8110860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 0x7fc0f8110320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0fd3c6700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fc0f000b970 tx=0x7fc0f000bc80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0f5ffb700 1 -- 172.21.15.90:0/2148598092 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f000d860 con 0x7fc0f8114c00 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0f5ffb700 1 -- 172.21.15.90:0/2148598092 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc0f000de80 con 0x7fc0f8114c00 2024-04-06T14:16:35.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.451+0000 7fc0f5ffb700 1 -- 172.21.15.90:0/2148598092 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0f0017400 con 0x7fc0f8114c00 2024-04-06T14:16:35.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.452+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0f81b2c30 con 0x7fc0f8114c00 2024-04-06T14:16:35.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.452+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0f81b30f0 con 0x7fc0f8114c00 2024-04-06T14:16:35.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.456+0000 7fc10afddb80 1 Processor -- start 2024-04-06T14:16:35.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 -- start start 2024-04-06T14:16:35.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea4202780 0x560ea4202b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea420f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea415a990 con 0x560ea420cde0 2024-04-06T14:16:35.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea4158350 con 0x560ea42030a0 2024-04-06T14:16:35.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc10afddb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea4153c40 con 0x560ea4202780 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.463+0000 7fc0fcbc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55202/0 (socket says 172.21.15.90:55202) 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/2961536853 learned_addr learned my addr 172.21.15.90:0/2961536853 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/2961536853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea4202780 msgr2=0x560ea4202b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea4202780 0x560ea4202b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/2961536853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 msgr2=0x560ea420f1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea420f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/2961536853 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ea3fd53b0 con 0x560ea42030a0 2024-04-06T14:16:35.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.464+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fc0ec00b6f0 tx=0x7fc0ec005c70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=634338c9177ef32a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.465+0000 7fc0e2ffd700 1 -- 172.21.15.90:0/2961536853 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0ec01d070 con 0x560ea42030a0 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.465+0000 7fc0e2ffd700 1 -- 172.21.15.90:0/2961536853 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc0ec01e040 con 0x560ea42030a0 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.465+0000 7fc0e2ffd700 1 -- 172.21.15.90:0/2961536853 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0ec004e00 con 0x560ea42030a0 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.466+0000 7fc10afddb80 1 -- 172.21.15.90:0/2961536853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 msgr2=0x560ea420c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.466+0000 7fc10afddb80 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fc0ec00b6f0 tx=0x7fc0ec005c70 comp rx=0 tx=0).stop 2024-04-06T14:16:35.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.466+0000 7fc0f5ffb700 1 -- 172.21.15.90:0/2148598092 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc0f000e630 con 0x7fc0f8114c00 2024-04-06T14:16:35.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.466+0000 7fc0f5ffb700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 0x7fc0e80a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc0f5ffb700 1 -- 172.21.15.90:0/2148598092 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc0f0006e40 con 0x7fc0f8114c00 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 -- 172.21.15.90:0/2961536853 shutdown_connections 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea420f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea4202780 0x560ea4202b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 --2- 172.21.15.90:0/2961536853 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea42030a0 0x560ea420c8a0 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 -- 172.21.15.90:0/2961536853 >> 172.21.15.90:0/2961536853 conn(0x560ea41616a0 msgr2=0x560ea41fda80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 0x7fc0e80a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 -- 172.21.15.90:0/2961536853 shutdown_connections 2024-04-06T14:16:35.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.467+0000 7fc10afddb80 1 -- 172.21.15.90:0/2961536853 wait complete. 2024-04-06T14:16:35.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.468+0000 7fc10afddb80 1 -- start start 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.468+0000 7fc10afddb80 1 Processor -- start 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea42030a0 0x560ea4209b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea420cde0 0x560ea420a260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea40efd80 con 0x560ea42030a0 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea40efa20 con 0x560ea4202780 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea3fd76a0 con 0x560ea420cde0 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc10afddb80 10 client.0 _ll_register_callbacks cb 0x560ea40f9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fd3c6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea420cde0 0x560ea420a260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55204/0 (socket says 172.21.15.90:55204) 2024-04-06T14:16:35.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/1732374827 learned_addr learned my addr 172.21.15.90:0/1732374827 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/1732374827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea420cde0 msgr2=0x560ea420a260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea420cde0 0x560ea420a260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/1732374827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea42030a0 msgr2=0x560ea4209b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea42030a0 0x560ea4209b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0fcbc5700 1 -- 172.21.15.90:0/1732374827 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0f810e480 con 0x560ea4202780 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.469+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 0x7fc0e80a3880 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x560ea42041f0 tx=0x7fc0e4004750 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.470+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fc0ec009ce0 tx=0x7fc0ec022630 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7762af358230c8ab server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.470+0000 7fc0d7fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.470+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0ec01d070 con 0x560ea4202780 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.470+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc0ec022c20 con 0x560ea4202780 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.470+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560ea4114380 con 0x560ea4202780 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.471+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc0ec022de0 con 0x560ea4202780 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.471+0000 7fc10afddb80 10 client.26975 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:35.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.471+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560ea3f81b60 con 0x560ea4202780 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.471+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560ea4114380 con 0x560ea4202780 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.471+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc0ec02c5c0 con 0x560ea4202780 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 10 client.26975 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 10 client.26975 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 20 client.26975 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:35.473 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4212460 0x560ea4214880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 4 client.26975 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:35.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560ea414c700 con 0x560ea4212460 2024-04-06T14:16:35.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.472+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4212460 0x560ea4214880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.473+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4212460 0x560ea4214880 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f42f8f94ec3c8cae server_cookie=59aeccc7df52ec67 in_seq=0 out_seq=0 2024-04-06T14:16:35.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.473+0000 7fc0d7fff700 10 client.26975 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:35.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.474+0000 7fc0d7fff700 1 -- 172.21.15.90:0/1732374827 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560ea414c700 con 0x560ea4212460 2024-04-06T14:16:35.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.474+0000 7fc0d7fff700 10 client.26975 handle_command_reply: tid=0 2024-04-06T14:16:35.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 client.26975 shutdown 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 msgr2=0x560ea420c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fc0ec009ce0 tx=0x7fc0ec022630 comp rx=0 tx=0).stop 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 shutdown_connections 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4212460 0x560ea4214880 crc :-1 s=READY pgs=193 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea420cde0 0x560ea420a260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea420c8a0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 --2- 172.21.15.90:0/1732374827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea42030a0 0x560ea4209b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.475+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 >> 172.21.15.90:0/1732374827 conn(0x560ea41616a0 msgr2=0x560ea41fcc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.477+0000 7fc0d7fff700 0 client.26975 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:35.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.477+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 shutdown_connections 2024-04-06T14:16:35.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.477+0000 7fc10afddb80 1 -- 172.21.15.90:0/1732374827 wait complete. 2024-04-06T14:16:35.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.477+0000 7fc10afddb80 20 client.26975 trim_cache size 0 max 16384 2024-04-06T14:16:35.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:35 smithi105 ceph-mon[46969]: pgmap v348: 97 pgs: 97 active+clean; 6.4 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 26 KiB/s rd, 22 MiB/s wr, 1.53k op/s 2024-04-06T14:16:35.503 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 Processor -- start 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 -- start start 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea4208b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea41fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea420aab0 con 0x560ea420cde0 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea4207fe0 con 0x560ea4202780 2024-04-06T14:16:35.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc10afddb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea420f1d0 con 0x560ea42030a0 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc0f7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc0f7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37910/0 (socket says 172.21.15.90:37910) 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3815234036 learned_addr learned my addr 172.21.15.90:0/3815234036 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc0fd3c6700 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea41fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.503+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea4208b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3815234036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 msgr2=0x560ea4208b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea4208b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3815234036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 msgr2=0x560ea41fd3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea41fd3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3815234036 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ea4119ea0 con 0x560ea42030a0 2024-04-06T14:16:35.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fc0f8111870 tx=0x7fc0e4014000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b5364e692bd56e11 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.504+0000 7fc0e37fe700 1 -- 172.21.15.90:0/3815234036 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0e40243b0 con 0x560ea42030a0 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.505+0000 7fc0e37fe700 1 -- 172.21.15.90:0/3815234036 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc0e4024040 con 0x560ea42030a0 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.505+0000 7fc0e37fe700 1 -- 172.21.15.90:0/3815234036 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0e4024e20 con 0x560ea42030a0 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.505+0000 7fc10afddb80 1 -- 172.21.15.90:0/3815234036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 msgr2=0x560ea420c3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.505+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fc0f8111870 tx=0x7fc0e4014000 comp rx=0 tx=0).stop 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 -- 172.21.15.90:0/3815234036 shutdown_connections 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea420cde0 0x560ea41fd3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea4202780 0x560ea4208b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3815234036 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea420c3d0 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 -- 172.21.15.90:0/3815234036 >> 172.21.15.90:0/3815234036 conn(0x560ea41616a0 msgr2=0x560ea42083f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 -- 172.21.15.90:0/3815234036 shutdown_connections 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 -- 172.21.15.90:0/3815234036 wait complete. 2024-04-06T14:16:35.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 -- start start 2024-04-06T14:16:35.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.506+0000 7fc10afddb80 1 Processor -- start 2024-04-06T14:16:35.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea4202780 0x560ea4221290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea420cde0 0x560ea4291da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea42220e0 con 0x560ea4202780 2024-04-06T14:16:35.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea42922e0 con 0x560ea420cde0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ea4292420 con 0x560ea42030a0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc10afddb80 10 client.0 _ll_register_callbacks cb 0x560ea40f9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:37914/0 (socket says 172.21.15.90:37914) 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3242756914 learned_addr learned my addr 172.21.15.90:0/3242756914 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea420cde0 msgr2=0x560ea4291da0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea420cde0 0x560ea4291da0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3242756914 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea4202780 msgr2=0x560ea4221290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea4202780 0x560ea4221290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 -- 172.21.15.90:0/3242756914 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0e4017070 con 0x560ea42030a0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0f7fff700 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc0e4009790 tx=0x7fc0e4003b10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=497519f928253eec server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.507+0000 7fc0d7fff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:35.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc0e4005230 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc0e4005390 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc0ec01e040 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc0e4005550 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc10afddb80 10 client.26758 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560ea3fd9cd0 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc0ec01e040 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.508+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc0e402a020 con 0x560ea42030a0 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 10 client.26758 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 10 client.26758 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 20 client.26758 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4293b40 0x560ea4295f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 4 client.26758 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x560ea3fa2cf0 con 0x560ea4293b40 2024-04-06T14:16:35.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4293b40 0x560ea4295f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc0fcbc5700 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4293b40 0x560ea4295f60 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1b9d7a2f910bfe35 server_cookie=9eed19d0c4c1d7fa in_seq=0 out_seq=0 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.509+0000 7fc0d7fff700 10 client.26758 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc0d7fff700 1 -- 172.21.15.90:0/3242756914 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x560ea3fa2cf0 con 0x560ea4293b40 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc0d7fff700 10 client.26758 handle_command_reply: tid=0 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 client.26758 shutdown 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 msgr2=0x560ea42217d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fc0e4009790 tx=0x7fc0e4003b10 comp rx=0 tx=0).stop 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 shutdown_connections 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560ea420cde0 0x560ea4291da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560ea4202780 0x560ea4221290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560ea4293b40 0x560ea4295f60 crc :-1 s=READY pgs=194 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 --2- 172.21.15.90:0/3242756914 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560ea42030a0 0x560ea42217d0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 >> 172.21.15.90:0/3242756914 conn(0x560ea41616a0 msgr2=0x560ea42287d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc0d7fff700 0 client.26758 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 shutdown_connections 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.510+0000 7fc10afddb80 1 -- 172.21.15.90:0/3242756914 wait complete. 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc10afddb80 20 client.26758 trim_cache size 0 max 16384 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:16:35.514 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 msgr2=0x7fc0e80a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 0x7fc0e80a3880 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x560ea42041f0 tx=0x7fc0e4004750 comp rx=0 tx=0).stop 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 msgr2=0x7fc0f81150b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fc0f000b970 tx=0x7fc0f000bc80 comp rx=0 tx=0).stop 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 shutdown_connections 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc0e80a13c0 0x7fc0e80a3880 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc0f8114c00 0x7fc0f81150b0 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc0f8069580 0x7fc0f8110320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 --2- 172.21.15.90:0/2148598092 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc0f8106090 0x7fc0f8110860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 >> 172.21.15.90:0/2148598092 conn(0x7fc0f8076db0 msgr2=0x7fc0f80fddf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 shutdown_connections 2024-04-06T14:16:35.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.511+0000 7fc0fee29700 1 -- 172.21.15.90:0/2148598092 wait complete. 2024-04-06T14:16:35.543 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:16:35.543 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:16:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.806+0000 7fc8232be700 1 -- 172.21.15.90:0/3078567297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 msgr2=0x7fc81c114960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.806+0000 7fc8232be700 1 --2- 172.21.15.90:0/3078567297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c114960 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7fc81400b0a0 tx=0x7fc81400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.808+0000 7fc8232be700 1 -- 172.21.15.90:0/3078567297 shutdown_connections 2024-04-06T14:16:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.808+0000 7fc8232be700 1 --2- 172.21.15.90:0/3078567297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c114960 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.808+0000 7fc8232be700 1 --2- 172.21.15.90:0/3078567297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c10dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.808+0000 7fc8232be700 1 --2- 172.21.15.90:0/3078567297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 0x7fc81c108ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.808+0000 7fc8232be700 1 -- 172.21.15.90:0/3078567297 >> 172.21.15.90:0/3078567297 conn(0x7fc81c06bd30 msgr2=0x7fc81c06c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:35.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.812+0000 7fc8232be700 1 -- 172.21.15.90:0/3078567297 shutdown_connections 2024-04-06T14:16:35.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.814+0000 7fc8232be700 1 -- 172.21.15.90:0/3078567297 wait complete. 2024-04-06T14:16:35.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.814+0000 7fc8232be700 1 Processor -- start 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.814+0000 7fc8232be700 1 -- start start 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 0x7fc81c19fa80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c19ffc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc81c1a4910 con 0x7fc81c10dbe0 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc81c1a4a80 con 0x7fc81c109000 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc8232be700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc81c1a4c40 con 0x7fc81c108640 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc82185b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc82185b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54570/0 (socket says 172.21.15.90:54570) 2024-04-06T14:16:35.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc82185b700 1 -- 172.21.15.90:0/2227196109 learned_addr learned my addr 172.21.15.90:0/2227196109 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc820859700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c19ffc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.815+0000 7fc82105a700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 0x7fc81c19fa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 -- 172.21.15.90:0/2227196109 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 msgr2=0x7fc81c19fa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 0x7fc81c19fa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 -- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 msgr2=0x7fc81c1a4340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 -- 172.21.15.90:0/2227196109 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc814009d00 con 0x7fc81c109000 2024-04-06T14:16:35.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc82185b700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc820859700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c19ffc0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fc81800b450 tx=0x7fc81800b760 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc818011630 con 0x7fc81c109000 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc818011c50 con 0x7fc81c109000 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc818010d00 con 0x7fc81c109000 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc81c1a4f20 con 0x7fc81c109000 2024-04-06T14:16:35.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.816+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc81c1ac7d0 con 0x7fc81c109000 2024-04-06T14:16:35.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.819+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc818010380 con 0x7fc81c109000 2024-04-06T14:16:35.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.819+0000 7fc8127fc700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 0x7fc8080a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:35.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.820+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc8180cc4e0 con 0x7fc81c109000 2024-04-06T14:16:35.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.821+0000 7fc82105a700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 0x7fc8080a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:35.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.822+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc81c06d090 con 0x7fc81c109000 2024-04-06T14:16:35.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.822+0000 7fc82105a700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 0x7fc8080a3790 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc80c006fd0 tx=0x7fc80c009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:35.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:35.828+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc818095a20 con 0x7fc81c109000 2024-04-06T14:16:36.204 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.204+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc81c1a0d30 con 0x7fc81c109000 2024-04-06T14:16:36.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.205+0000 7fc8127fc700 1 -- 172.21.15.90:0/2227196109 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fc818015090 con 0x7fc81c109000 2024-04-06T14:16:36.208 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:16:36.208 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:16:36.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 msgr2=0x7fc8080a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 0x7fc8080a3790 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc80c006fd0 tx=0x7fc80c009190 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 msgr2=0x7fc81c19ffc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c19ffc0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7fc81800b450 tx=0x7fc81800b760 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 shutdown_connections 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc8080a12d0 0x7fc8080a3790 secure :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc80c006fd0 tx=0x7fc80c009190 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc81c10dbe0 0x7fc81c1a4340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc81c108640 0x7fc81c19fa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 --2- 172.21.15.90:0/2227196109 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc81c109000 0x7fc81c19ffc0 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 >> 172.21.15.90:0/2227196109 conn(0x7fc81c06bd30 msgr2=0x7fc81c070370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 shutdown_connections 2024-04-06T14:16:36.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.211+0000 7fc8232be700 1 -- 172.21.15.90:0/2227196109 wait complete. 2024-04-06T14:16:36.213 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:16:36.228 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:36.228 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:16:36.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- 172.21.15.90:0/1889223819 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa434104f90 msgr2=0x7fa43410cb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1889223819 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa434104f90 0x7fa43410cb10 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fa42c009f90 tx=0x7fa42c00b330 comp rx=0 tx=0).stop 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- 172.21.15.90:0/1889223819 shutdown_connections 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1889223819 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa434104f90 0x7fa43410cb10 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1889223819 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa4340ffe70 0x7fa434100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1889223819 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434100820 0x7fa434104860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- 172.21.15.90:0/1889223819 >> 172.21.15.90:0/1889223819 conn(0x7fa434076d80 msgr2=0x7fa434077190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- 172.21.15.90:0/1889223819 shutdown_connections 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- 172.21.15.90:0/1889223819 wait complete. 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 Processor -- start 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.401+0000 7fa43c58a700 1 -- start start 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4340ffe70 0x7fa434108ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa434104f90 0x7fa4341091e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa43410cea0 con 0x7fa434109720 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4341a9fd0 con 0x7fa434104f90 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43c58a700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa4341aa160 con 0x7fa4340ffe70 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54584/0 (socket says 172.21.15.90:54584) 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 -- 172.21.15.90:0/1880232811 learned_addr learned my addr 172.21.15.90:0/1880232811 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 -- 172.21.15.90:0/1880232811 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4340ffe70 msgr2=0x7fa434108ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4340ffe70 0x7fa434108ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 -- 172.21.15.90:0/1880232811 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa434104f90 msgr2=0x7fa4341091e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa434104f90 0x7fa4341091e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 -- 172.21.15.90:0/1880232811 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa42c009c40 con 0x7fa434109720 2024-04-06T14:16:36.404 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.402+0000 7fa43ab27700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7fa42c007cd0 tx=0x7fa42c007db0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.403+0000 7fa42b7fe700 1 -- 172.21.15.90:0/1880232811 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa42c00e040 con 0x7fa434109720 2024-04-06T14:16:36.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.403+0000 7fa42b7fe700 1 -- 172.21.15.90:0/1880232811 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa42c007870 con 0x7fa434109720 2024-04-06T14:16:36.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.403+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa4341aa3e0 con 0x7fa434109720 2024-04-06T14:16:36.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.403+0000 7fa42b7fe700 1 -- 172.21.15.90:0/1880232811 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa42c01caa0 con 0x7fa434109720 2024-04-06T14:16:36.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.403+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4341aa860 con 0x7fa434109720 2024-04-06T14:16:36.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.408+0000 7fa44873eb80 1 Processor -- start 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 -- start start 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d1780 0x561fae7d1b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 0x561fae7db8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 0x561fae7de1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae729990 con 0x561fae7dbde0 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae727350 con 0x561fae7d20a0 2024-04-06T14:16:36.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.409+0000 7fa44873eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae722c40 con 0x561fae7d1780 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.418+0000 7fa42b7fe700 1 -- 172.21.15.90:0/1880232811 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa42c01a600 con 0x7fa434109720 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.418+0000 7fa42b7fe700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 0x7fa4200a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.419+0000 7fa42b7fe700 1 -- 172.21.15.90:0/1880232811 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa42c0cd9a0 con 0x7fa434109720 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.423+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 0x561fae7de1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.423+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 0x561fae7de1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54590/0 (socket says 172.21.15.90:54590) 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.423+0000 7fa439b25700 1 -- 172.21.15.90:0/2524248306 learned_addr learned my addr 172.21.15.90:0/2524248306 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.423+0000 7fa439b25700 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d1780 0x561fae7d1b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.425+0000 7fa43a326700 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 0x561fae7db8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.432+0000 7fa43ab27700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 0x7fa4200a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.433+0000 7fa43a326700 1 -- 172.21.15.90:0/2524248306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d1780 msgr2=0x561fae7d1b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.433+0000 7fa43a326700 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d1780 0x561fae7d1b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.433+0000 7fa43a326700 1 -- 172.21.15.90:0/2524248306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 msgr2=0x561fae7de1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.433+0000 7fa43a326700 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 0x561fae7de1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.433+0000 7fa43a326700 1 -- 172.21.15.90:0/2524248306 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561fae5a43b0 con 0x561fae7d20a0 2024-04-06T14:16:36.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.434+0000 7fa43a326700 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 0x561fae7db8a0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa4300097b0 tx=0x7fa430009ac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=39506070b29671c1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.447 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.446+0000 7fa428ff9700 1 -- 172.21.15.90:0/2524248306 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa430011830 con 0x561fae7d20a0 2024-04-06T14:16:36.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.446+0000 7fa428ff9700 1 -- 172.21.15.90:0/2524248306 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa430011e50 con 0x561fae7d20a0 2024-04-06T14:16:36.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.446+0000 7fa428ff9700 1 -- 172.21.15.90:0/2524248306 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa43000b360 con 0x561fae7d20a0 2024-04-06T14:16:36.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.447+0000 7fa44873eb80 1 -- 172.21.15.90:0/2524248306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 msgr2=0x561fae7db8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.447+0000 7fa44873eb80 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 0x561fae7db8a0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fa4300097b0 tx=0x7fa430009ac0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.450+0000 7fa43ab27700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 0x7fa4200a3850 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fa42c0a2980 tx=0x7fa42c009920 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.451+0000 7fa44873eb80 1 -- 172.21.15.90:0/2524248306 shutdown_connections 2024-04-06T14:16:36.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.451+0000 7fa44873eb80 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7dbde0 0x561fae7de1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.451+0000 7fa44873eb80 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d1780 0x561fae7d1b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.451+0000 7fa44873eb80 1 --2- 172.21.15.90:0/2524248306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d20a0 0x561fae7db8a0 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.451+0000 7fa44873eb80 1 -- 172.21.15.90:0/2524248306 >> 172.21.15.90:0/2524248306 conn(0x561fae7306a0 msgr2=0x561fae7cca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.462+0000 7fa44873eb80 1 -- 172.21.15.90:0/2524248306 shutdown_connections 2024-04-06T14:16:36.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.474+0000 7fa44873eb80 1 -- 172.21.15.90:0/2524248306 wait complete. 2024-04-06T14:16:36.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.474+0000 7fa44873eb80 1 -- start start 2024-04-06T14:16:36.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.474+0000 7fa44873eb80 1 Processor -- start 2024-04-06T14:16:36.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d1780 0x561fae7db810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7d9290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae6bed80 con 0x561fae7d20a0 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae6bea20 con 0x561fae7d1780 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae5a66a0 con 0x561fae7dbde0 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.475+0000 7fa44873eb80 10 client.0 _ll_register_callbacks cb 0x561fae6c8470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.476+0000 7fa43a326700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.476+0000 7fa43a326700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54598/0 (socket says 172.21.15.90:54598) 2024-04-06T14:16:36.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.476+0000 7fa43a326700 1 -- 172.21.15.90:0/3195741197 learned_addr learned my addr 172.21.15.90:0/3195741197 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.480+0000 7fa43a326700 1 -- 172.21.15.90:0/3195741197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 msgr2=0x561fae7d9290 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.480+0000 7fa43a326700 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7d9290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.480+0000 7fa43a326700 1 -- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d1780 msgr2=0x561fae7db810 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.480+0000 7fa43a326700 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d1780 0x561fae7db810 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.480+0000 7fa43a326700 1 -- 172.21.15.90:0/3195741197 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa4300093e0 con 0x561fae7d20a0 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa43a326700 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7fa4300105b0 tx=0x7fa430010690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c02513dd83e56c1c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa41dffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa430011630 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa430011790 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x561fae6e3380 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.481+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa430011950 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 10 client.17286 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561fae550b60 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x561fae6e3380 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa430023c80 con 0x561fae7d20a0 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 10 client.17286 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 10 client.17286 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:16:36.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 20 client.17286 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:36.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae7fc9a0 0x561fae7fedc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 4 client.17286 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:36.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.482+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x561fae71b700 con 0x561fae7fc9a0 2024-04-06T14:16:36.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.483+0000 7fa439b25700 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae7fc9a0 0x561fae7fedc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.486+0000 7fa439b25700 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae7fc9a0 0x561fae7fedc0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=63eac2798362125b server_cookie=d8e1d6d4c23a8787 in_seq=0 out_seq=0 2024-04-06T14:16:36.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.486+0000 7fa41dffb700 10 client.17286 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.496+0000 7fa41dffb700 1 -- 172.21.15.90:0/3195741197 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x561fae71b700 con 0x561fae7fc9a0 2024-04-06T14:16:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.496+0000 7fa41dffb700 10 client.17286 handle_command_reply: tid=0 2024-04-06T14:16:36.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.497+0000 7fa44873eb80 1 client.17286 shutdown 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 msgr2=0x561fae7d8b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 secure :-1 s=READY pgs=750 cs=0 l=1 rev1=1 crypto rx=0x7fa4300105b0 tx=0x7fa430010690 comp rx=0 tx=0).stop 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 shutdown_connections 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae7fc9a0 0x561fae7fedc0 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7d9290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7d1780 0x561fae7db810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3195741197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7d8b50 unknown :-1 s=CLOSED pgs=750 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa41dffb700 0 client.17286 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 >> 172.21.15.90:0/3195741197 conn(0x561fae7306a0 msgr2=0x561fae7cbc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 shutdown_connections 2024-04-06T14:16:36.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.504+0000 7fa44873eb80 1 -- 172.21.15.90:0/3195741197 wait complete. 2024-04-06T14:16:36.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.505+0000 7fa44873eb80 20 client.17286 trim_cache size 0 max 16384 2024-04-06T14:16:36.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.526+0000 7fa44873eb80 1 Processor -- start 2024-04-06T14:16:36.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 -- start start 2024-04-06T14:16:36.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d20a0 0x561fae7d3ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7dbde0 0x561fae7d43e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae71b700 con 0x561fae7d1780 2024-04-06T14:16:36.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae7c6d20 con 0x561fae7dbde0 2024-04-06T14:16:36.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.538+0000 7fa44873eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae7cd7c0 con 0x561fae7d20a0 2024-04-06T14:16:36.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.545+0000 7fa43a326700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d20a0 0x561fae7d3ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.545+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.545+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54604/0 (socket says 172.21.15.90:54604) 2024-04-06T14:16:36.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.545+0000 7fa439b25700 1 -- 172.21.15.90:0/3316380883 learned_addr learned my addr 172.21.15.90:0/3316380883 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.550+0000 7fa439b25700 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7dbde0 0x561fae7d43e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.555+0000 7fa439b25700 1 -- 172.21.15.90:0/3316380883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d20a0 msgr2=0x561fae7d3ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.556+0000 7fa439b25700 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d20a0 0x561fae7d3ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.556+0000 7fa439b25700 1 -- 172.21.15.90:0/3316380883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7dbde0 msgr2=0x561fae7d43e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.556+0000 7fa439b25700 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7dbde0 0x561fae7d43e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.556+0000 7fa439b25700 1 -- 172.21.15.90:0/3316380883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x561fae6e8ea0 con 0x561fae7d1780 2024-04-06T14:16:36.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.557+0000 7fa439b25700 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7fa424004790 tx=0x7fa4240096b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=edce3d2d46ef7c12 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.567+0000 7fa4297fa700 1 -- 172.21.15.90:0/3316380883 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa42401ad70 con 0x561fae7d1780 2024-04-06T14:16:36.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.567+0000 7fa4297fa700 1 -- 172.21.15.90:0/3316380883 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa424019070 con 0x561fae7d1780 2024-04-06T14:16:36.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.567+0000 7fa4297fa700 1 -- 172.21.15.90:0/3316380883 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa424011440 con 0x561fae7d1780 2024-04-06T14:16:36.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 -- 172.21.15.90:0/3316380883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 msgr2=0x561fae7d3630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 secure :-1 s=READY pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7fa424004790 tx=0x7fa4240096b0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 -- 172.21.15.90:0/3316380883 shutdown_connections 2024-04-06T14:16:36.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7dbde0 0x561fae7d43e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d1780 0x561fae7d3630 secure :-1 s=CLOSED pgs=751 cs=0 l=1 rev1=1 crypto rx=0x7fa424004790 tx=0x7fa4240096b0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 --2- 172.21.15.90:0/3316380883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7d20a0 0x561fae7d3ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.570+0000 7fa44873eb80 1 -- 172.21.15.90:0/3316380883 >> 172.21.15.90:0/3316380883 conn(0x561fae7306a0 msgr2=0x561fae8036e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.574+0000 7fa44873eb80 1 -- 172.21.15.90:0/3316380883 shutdown_connections 2024-04-06T14:16:36.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.574+0000 7fa44873eb80 1 -- 172.21.15.90:0/3316380883 wait complete. 2024-04-06T14:16:36.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.574+0000 7fa44873eb80 1 -- start start 2024-04-06T14:16:36.574 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.574+0000 7fa44873eb80 1 Processor -- start 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7e0b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7e10a0 0x561fae7e1520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae7e2f40 con 0x561fae7d20a0 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae8507a0 con 0x561fae7e10a0 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x561fae8508e0 con 0x561fae7dbde0 2024-04-06T14:16:36.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.575+0000 7fa44873eb80 10 client.0 _ll_register_callbacks cb 0x561fae6c8470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:36.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.576+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.576+0000 7fa439b25700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54608/0 (socket says 172.21.15.90:54608) 2024-04-06T14:16:36.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.576+0000 7fa439b25700 1 -- 172.21.15.90:0/1188417298 learned_addr learned my addr 172.21.15.90:0/1188417298 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.576+0000 7fa439b25700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7e10a0 0x561fae7e1520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.577+0000 7fa43a326700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7e0b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 -- 172.21.15.90:0/1188417298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 msgr2=0x561fae7e0b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7e0b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 -- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7e10a0 msgr2=0x561fae7e1520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7e10a0 0x561fae7e1520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 -- 172.21.15.90:0/1188417298 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa424009e70 con 0x561fae7d20a0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa439b25700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7fa424003fd0 tx=0x7fa424003bf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9b0282c717aadf4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa41dffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa424004000 con 0x561fae7d20a0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa424004160 con 0x561fae7d20a0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa430011e50 con 0x561fae7d20a0 2024-04-06T14:16:36.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa424004320 con 0x561fae7d20a0 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa44873eb80 10 client.17292 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.578+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x561fae5a8cd0 con 0x561fae7d20a0 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa430011e50 con 0x561fae7d20a0 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa424025070 con 0x561fae7d20a0 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 10 client.17292 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 10 client.17292 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 20 client.17292 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:36.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae851e80 0x561fae8542a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 4 client.17292 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:36.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.579+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x561fae571cf0 con 0x561fae851e80 2024-04-06T14:16:36.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.582+0000 7fa43a326700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae851e80 0x561fae8542a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa43a326700 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae851e80 0x561fae8542a0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=d747d06f7a6ebc0b server_cookie=cd802fa87f74637d in_seq=0 out_seq=0 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa41dffb700 10 client.17292 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa41dffb700 1 -- 172.21.15.90:0/1188417298 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x561fae571cf0 con 0x561fae851e80 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa41dffb700 10 client.17292 handle_command_reply: tid=0 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 client.17292 shutdown 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 msgr2=0x561fae7e2810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 secure :-1 s=READY pgs=752 cs=0 l=1 rev1=1 crypto rx=0x7fa424003fd0 tx=0x7fa424003bf0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 shutdown_connections 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x561fae851e80 0x561fae8542a0 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x561fae7e10a0 0x561fae7e1520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x561fae7d20a0 0x561fae7e2810 unknown :-1 s=CLOSED pgs=752 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa41dffb700 0 client.17292 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 --2- 172.21.15.90:0/1188417298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x561fae7dbde0 0x561fae7e0b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 >> 172.21.15.90:0/1188417298 conn(0x561fae7306a0 msgr2=0x561fae7d3ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.583+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 shutdown_connections 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa44873eb80 1 -- 172.21.15.90:0/1188417298 wait complete. 2024-04-06T14:16:36.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa44873eb80 20 client.17292 trim_cache size 0 max 16384 2024-04-06T14:16:36.586 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:36.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 msgr2=0x7fa4200a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 0x7fa4200a3850 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fa42c0a2980 tx=0x7fa42c009920 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 msgr2=0x7fa4341a9a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 secure :-1 s=READY pgs=749 cs=0 l=1 rev1=1 crypto rx=0x7fa42c007cd0 tx=0x7fa42c007db0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 shutdown_connections 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa4200a1390 0x7fa4200a3850 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa434109720 0x7fa4341a9a90 unknown :-1 s=CLOSED pgs=749 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa4340ffe70 0x7fa434108ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 --2- 172.21.15.90:0/1880232811 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa434104f90 0x7fa4341091e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 >> 172.21.15.90:0/1880232811 conn(0x7fa434076d80 msgr2=0x7fa4340ff320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 shutdown_connections 2024-04-06T14:16:36.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.584+0000 7fa43c58a700 1 -- 172.21.15.90:0/1880232811 wait complete. 2024-04-06T14:16:36.604 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:36.604 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:16:36.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 -- 172.21.15.90:0/1168401216 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc104f70 msgr2=0x7fc3fc10caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 --2- 172.21.15.90:0/1168401216 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc104f70 0x7fc3fc10caf0 secure :-1 s=READY pgs=753 cs=0 l=1 rev1=1 crypto rx=0x7fc3f400b0a0 tx=0x7fc3f400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 -- 172.21.15.90:0/1168401216 shutdown_connections 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 --2- 172.21.15.90:0/1168401216 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc104f70 0x7fc3fc10caf0 unknown :-1 s=CLOSED pgs=753 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 --2- 172.21.15.90:0/1168401216 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 --2- 172.21.15.90:0/1168401216 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc100820 0x7fc3fc104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 -- 172.21.15.90:0/1168401216 >> 172.21.15.90:0/1168401216 conn(0x7fc3fc076d50 msgr2=0x7fc3fc077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 -- 172.21.15.90:0/1168401216 shutdown_connections 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.764+0000 7fc4025b8700 1 -- 172.21.15.90:0/1168401216 wait complete. 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 Processor -- start 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 -- start start 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc198120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc198ba0 0x7fc3fc1a31a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3fc1a3830 con 0x7fc3fc0ffe70 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3fc1a39a0 con 0x7fc3fc198ba0 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc4025b8700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3fc1a3b60 con 0x7fc3fc104f70 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3fbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc198120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc400b55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc198ba0 0x7fc3fc1a31a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3fbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc198120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54628/0 (socket says 172.21.15.90:54628) 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38010/0 (socket says 172.21.15.90:38010) 2024-04-06T14:16:36.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3fbfff700 1 -- 172.21.15.90:0/4154669569 learned_addr learned my addr 172.21.15.90:0/4154669569 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 -- 172.21.15.90:0/4154669569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc198ba0 msgr2=0x7fc3fc1a31a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc198ba0 0x7fc3fc1a31a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 -- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 msgr2=0x7fc3fc198120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc198120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.765+0000 7fc3f3fff700 1 -- 172.21.15.90:0/4154669569 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3f4009d00 con 0x7fc3fc104f70 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fc3e400e9c0 tx=0x7fc3e400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc3f9ffb700 1 -- 172.21.15.90:0/4154669569 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e400c9c0 con 0x7fc3fc104f70 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc3f9ffb700 1 -- 172.21.15.90:0/4154669569 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3e4010430 con 0x7fc3fc104f70 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc3f9ffb700 1 -- 172.21.15.90:0/4154669569 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e4018680 con 0x7fc3fc104f70 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3fc195f70 con 0x7fc3fc104f70 2024-04-06T14:16:36.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.768+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3fc196430 con 0x7fc3fc104f70 2024-04-06T14:16:36.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.771+0000 7fc3f9ffb700 1 -- 172.21.15.90:0/4154669569 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc3e400cb20 con 0x7fc3fc104f70 2024-04-06T14:16:36.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.771+0000 7fc3f9ffb700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 0x7fc3e80a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.771+0000 7fc3f9ffb700 1 -- 172.21.15.90:0/4154669569 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc3e4014070 con 0x7fc3fc104f70 2024-04-06T14:16:36.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.772+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 0x7fc3e80a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.772+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 0x7fc3e80a38a0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fc3ec010b30 tx=0x7fc3ec012040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.772+0000 7fc40e76cb80 1 Processor -- start 2024-04-06T14:16:36.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.773+0000 7fc40e76cb80 1 -- start start 2024-04-06T14:16:36.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda03db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda04a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9f95990 con 0x55bdda03e0a0 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9f93350 con 0x55bdda03d780 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9f8ec40 con 0x55bdda047de0 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54638/0 (socket says 172.21.15.90:54638) 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 -- 172.21.15.90:0/580280129 learned_addr learned my addr 172.21.15.90:0/580280129 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc400b55700 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda04a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda03db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 -- 172.21.15.90:0/580280129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 msgr2=0x55bdda04a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda04a1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 -- 172.21.15.90:0/580280129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 msgr2=0x55bdda03db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda03db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 -- 172.21.15.90:0/580280129 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bdd9e103b0 con 0x55bdda03e0a0 2024-04-06T14:16:36.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.774+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7fc3e40a1c40 tx=0x7fc3e40996d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a10753e8cc10ed12 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc3f27fc700 1 -- 172.21.15.90:0/580280129 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e4099cf0 con 0x55bdda03e0a0 2024-04-06T14:16:36.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc3f27fc700 1 -- 172.21.15.90:0/580280129 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3e409db20 con 0x55bdda03e0a0 2024-04-06T14:16:36.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc3f27fc700 1 -- 172.21.15.90:0/580280129 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e4099440 con 0x55bdda03e0a0 2024-04-06T14:16:36.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 -- 172.21.15.90:0/580280129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 msgr2=0x55bdda0478a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 secure :-1 s=READY pgs=754 cs=0 l=1 rev1=1 crypto rx=0x7fc3e40a1c40 tx=0x7fc3e40996d0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 -- 172.21.15.90:0/580280129 shutdown_connections 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda04a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda03db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/580280129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda0478a0 unknown :-1 s=CLOSED pgs=754 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 -- 172.21.15.90:0/580280129 >> 172.21.15.90:0/580280129 conn(0x55bdd9f9c6a0 msgr2=0x55bdda038a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 -- 172.21.15.90:0/580280129 shutdown_connections 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.775+0000 7fc40e76cb80 1 -- 172.21.15.90:0/580280129 wait complete. 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 -- start start 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 Processor -- start 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda0478a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda044c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda0453d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9f2ad80 con 0x55bdda03e0a0 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9f2aa20 con 0x55bdda03d780 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdd9e126a0 con 0x55bdda047de0 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc40e76cb80 10 client.0 _ll_register_callbacks cb 0x55bdd9f34470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda044c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda044c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54648/0 (socket says 172.21.15.90:54648) 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc3f3fff700 1 -- 172.21.15.90:0/3370699963 learned_addr learned my addr 172.21.15.90:0/3370699963 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.776+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda0478a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc400b55700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda0453d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 -- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 msgr2=0x55bdda0453d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda0453d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 -- 172.21.15.90:0/3370699963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 msgr2=0x55bdda044c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda044c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 -- 172.21.15.90:0/3370699963 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3fc196430 con 0x55bdda03d780 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.777+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda0478a0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fc3ec010b00 tx=0x7fc3ec0047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e98a2eacb7d7850d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc3db7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3ec01f050 con 0x55bdda03d780 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc3ec026410 con 0x55bdda03d780 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bdd9f4f380 con 0x55bdda03d780 2024-04-06T14:16:36.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3ec0265d0 con 0x55bdda03d780 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc40e76cb80 10 client.27005 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bdd9dbcb60 con 0x55bdda03d780 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.778+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55bdd9f4f380 con 0x55bdda03d780 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3ec024760 con 0x55bdda03d780 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 10 client.27005 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 10 client.27005 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 20 client.27005 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda057ad0 0x55bdda059ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 4 client.27005 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bdd9f87700 con 0x55bdda057ad0 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda057ad0 0x55bdda059ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda057ad0 0x55bdda059ef0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ba1825332606e777 server_cookie=6105e8e291e2e4bd in_seq=0 out_seq=0 2024-04-06T14:16:36.781 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.779+0000 7fc3db7fe700 10 client.27005 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:36.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3370699963 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55bdd9f87700 con 0x55bdda057ad0 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc3db7fe700 10 client.27005 handle_command_reply: tid=0 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 client.27005 shutdown 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 msgr2=0x55bdda0478a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda0478a0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fc3ec010b00 tx=0x7fc3ec0047c0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 shutdown_connections 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda057ad0 0x55bdda059ef0 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda047de0 0x55bdda0453d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda03d780 0x55bdda0478a0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3370699963 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda044c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc3db7fe700 0 client.27005 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 >> 172.21.15.90:0/3370699963 conn(0x55bdd9f9c6a0 msgr2=0x55bdda037c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 shutdown_connections 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.780+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3370699963 wait complete. 2024-04-06T14:16:36.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.781+0000 7fc40e76cb80 20 client.27005 trim_cache size 0 max 16384 2024-04-06T14:16:36.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.796+0000 7fc40e76cb80 1 Processor -- start 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.796+0000 7fc40e76cb80 1 -- start start 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03d780 0x55bdda059ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda038a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda039010 con 0x55bdda03e0a0 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda0431c0 con 0x55bdda047de0 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda0459b0 con 0x55bdda03d780 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54660/0 (socket says 172.21.15.90:54660) 2024-04-06T14:16:36.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3fbfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03d780 0x55bdda059ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 -- 172.21.15.90:0/947689793 learned_addr learned my addr 172.21.15.90:0/947689793 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc400b55700 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda038a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 -- 172.21.15.90:0/947689793 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03d780 msgr2=0x55bdda059ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03d780 0x55bdda059ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 -- 172.21.15.90:0/947689793 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 msgr2=0x55bdda038a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda038a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 -- 172.21.15.90:0/947689793 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bdd9f54ea0 con 0x55bdda03e0a0 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7fc3e40999f0 tx=0x7fc3e40af040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c9a95b5bf4b758b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f2ffd700 1 -- 172.21.15.90:0/947689793 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e4098090 con 0x55bdda03e0a0 2024-04-06T14:16:36.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f2ffd700 1 -- 172.21.15.90:0/947689793 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3e40991e0 con 0x55bdda03e0a0 2024-04-06T14:16:36.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.797+0000 7fc3f2ffd700 1 -- 172.21.15.90:0/947689793 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e40a1ce0 con 0x55bdda03e0a0 2024-04-06T14:16:36.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.798+0000 7fc40e76cb80 1 -- 172.21.15.90:0/947689793 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 msgr2=0x55bdda038520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.798+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 secure :-1 s=READY pgs=755 cs=0 l=1 rev1=1 crypto rx=0x7fc3e40999f0 tx=0x7fc3e40af040 comp rx=0 tx=0).stop 2024-04-06T14:16:36.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.798+0000 7fc40e76cb80 1 -- 172.21.15.90:0/947689793 shutdown_connections 2024-04-06T14:16:36.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda038a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03d780 0x55bdda059ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/947689793 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03e0a0 0x55bdda038520 unknown :-1 s=CLOSED pgs=755 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 -- 172.21.15.90:0/947689793 >> 172.21.15.90:0/947689793 conn(0x55bdd9f9c6a0 msgr2=0x55bdda05e810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 -- 172.21.15.90:0/947689793 shutdown_connections 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 -- 172.21.15.90:0/947689793 wait complete. 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.799+0000 7fc40e76cb80 1 -- start start 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 Processor -- start 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 0x55bdda04c940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 0x55bdda04ce80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda0aa8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda0aae00 con 0x55bdda03d780 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda0aaf40 con 0x55bdda047de0 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bdda0ab080 con 0x55bdda03e0a0 2024-04-06T14:16:36.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc40e76cb80 10 client.0 _ll_register_callbacks cb 0x55bdd9f34470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc3fbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 0x55bdda04c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc3f3fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 0x55bdda04ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc400b55700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda0aa8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc3fbfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 0x55bdda04c940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54664/0 (socket says 172.21.15.90:54664) 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.800+0000 7fc3fbfff700 1 -- 172.21.15.90:0/3966568539 learned_addr learned my addr 172.21.15.90:0/3966568539 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 -- 172.21.15.90:0/3966568539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 msgr2=0x55bdda0aa8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda0aa8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 -- 172.21.15.90:0/3966568539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 msgr2=0x55bdda04c940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 0x55bdda04c940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 -- 172.21.15.90:0/3966568539 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55bdd9e103b0 con 0x55bdda03e0a0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3f3fff700 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 0x55bdda04ce80 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x55bdda05a400 tx=0x7fc3e40a2290 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e9486fb2801c6286 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3e4098090 con 0x55bdda03e0a0 2024-04-06T14:16:36.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc3e409db20 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc3e4099f50 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3e409a110 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 10 client.26791 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55bdd9e14cd0 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3e409db20 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc3e4095ae0 con 0x55bdda03e0a0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 10 client.26791 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 10 client.26791 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 20 client.26791 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda0ac320 0x55bdda0ae710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.801+0000 7fc40e76cb80 4 client.26791 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55bdd9dddcf0 con 0x55bdda0ac320 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda0ac320 0x55bdda0ae710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3fbfff700 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda0ac320 0x55bdda0ae710 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=3b577084e5db559d server_cookie=da01ca596f13ba7b in_seq=0 out_seq=0 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3db7fe700 10 client.26791 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3db7fe700 1 -- 172.21.15.90:0/3966568539 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55bdd9dddcf0 con 0x55bdda0ac320 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3db7fe700 10 client.26791 handle_command_reply: tid=0 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 client.26791 shutdown 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 msgr2=0x55bdda04ce80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 0x55bdda04ce80 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x55bdda05a400 tx=0x7fc3e40a2290 comp rx=0 tx=0).stop 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 shutdown_connections 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55bdda0ac320 0x55bdda0ae710 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55bdda047de0 0x55bdda0aa8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc3db7fe700 0 client.26791 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55bdda03d780 0x55bdda04c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.802+0000 7fc40e76cb80 1 --2- 172.21.15.90:0/3966568539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55bdda03e0a0 0x55bdda04ce80 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 >> 172.21.15.90:0/3966568539 conn(0x55bdd9f9c6a0 msgr2=0x55bdda03ff40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 shutdown_connections 2024-04-06T14:16:36.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc40e76cb80 1 -- 172.21.15.90:0/3966568539 wait complete. 2024-04-06T14:16:36.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc40e76cb80 20 client.26791 trim_cache size 0 max 16384 2024-04-06T14:16:36.811 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 msgr2=0x7fc3e80a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 0x7fc3e80a38a0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fc3ec010b30 tx=0x7fc3ec012040 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 msgr2=0x7fc3fc198660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fc3e400e9c0 tx=0x7fc3e400ecd0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 shutdown_connections 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc3e80a13e0 0x7fc3e80a38a0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc3fc198ba0 0x7fc3fc1a31a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc3fc0ffe70 0x7fc3fc198120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 --2- 172.21.15.90:0/4154669569 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc3fc104f70 0x7fc3fc198660 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 >> 172.21.15.90:0/4154669569 conn(0x7fc3fc076d50 msgr2=0x7fc3fc0fdd30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 shutdown_connections 2024-04-06T14:16:36.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.803+0000 7fc4025b8700 1 -- 172.21.15.90:0/4154669569 wait complete. 2024-04-06T14:16:36.818 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:16:36.818 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:16:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 -- 172.21.15.90:0/3203711517 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0908104f70 msgr2=0x7f090810caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 --2- 172.21.15.90:0/3203711517 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0908104f70 0x7f090810caf0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f090000b0a0 tx=0x7f090000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 -- 172.21.15.90:0/3203711517 shutdown_connections 2024-04-06T14:16:36.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 --2- 172.21.15.90:0/3203711517 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0908104f70 0x7f090810caf0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 --2- 172.21.15.90:0/3203711517 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f09080ffe70 0x7f0908100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 --2- 172.21.15.90:0/3203711517 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0908100820 0x7f0908104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 -- 172.21.15.90:0/3203711517 >> 172.21.15.90:0/3203711517 conn(0x7f0908076d50 msgr2=0x7f0908077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.974+0000 7f090e206700 1 -- 172.21.15.90:0/3203711517 shutdown_connections 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.975+0000 7f090e206700 1 -- 172.21.15.90:0/3203711517 wait complete. 2024-04-06T14:16:36.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.975+0000 7f090e206700 1 Processor -- start 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 -- start start 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f09080ffe70 0x7f090810d430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f090819f180 0x7f090810deb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09081a9f80 con 0x7f090819f180 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09081aa0f0 con 0x7f0908104f70 2024-04-06T14:16:36.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f090e206700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f09081aa2b0 con 0x7f09080ffe70 2024-04-06T14:16:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55348/0 (socket says 172.21.15.90:55348) 2024-04-06T14:16:36.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f0906ffd700 1 -- 172.21.15.90:0/1575150083 learned_addr learned my addr 172.21.15.90:0/1575150083 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.976+0000 7f09077fe700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f09080ffe70 0x7f090810d430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 -- 172.21.15.90:0/1575150083 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f09080ffe70 msgr2=0x7f090810d430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f09080ffe70 0x7f090810d430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 -- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f090819f180 msgr2=0x7f090810deb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f090819f180 0x7f090810deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 -- 172.21.15.90:0/1575150083 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0900009d00 con 0x7f0908104f70 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f08fc00b500 tx=0x7f08fc00b8c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.977+0000 7f0904ff9700 1 -- 172.21.15.90:0/1575150083 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc011830 con 0x7f0908104f70 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.978+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f090819f710 con 0x7f0908104f70 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.978+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f090819fc70 con 0x7f0908104f70 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.978+0000 7f0904ff9700 1 -- 172.21.15.90:0/1575150083 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f08fc011e50 con 0x7f0908104f70 2024-04-06T14:16:36.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.978+0000 7f0904ff9700 1 -- 172.21.15.90:0/1575150083 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc00f2e0 con 0x7f0908104f70 2024-04-06T14:16:36.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.980+0000 7f0904ff9700 1 -- 172.21.15.90:0/1575150083 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f08fc00f440 con 0x7f0908104f70 2024-04-06T14:16:36.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.981+0000 7f0904ff9700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 0x7f08f00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.981+0000 7f09077fe700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 0x7f08f00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.981+0000 7f0904ff9700 1 -- 172.21.15.90:0/1575150083 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f08fc0cc750 con 0x7f0908104f70 2024-04-06T14:16:36.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.981+0000 7f091a3bab80 1 Processor -- start 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.981+0000 7f091a3bab80 1 -- start start 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb288780 0x5642cb288b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2951d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb1e0990 con 0x5642cb292de0 2024-04-06T14:16:36.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb1de350 con 0x5642cb2890a0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f091a3bab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb1d9c40 con 0x5642cb288780 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f09077fe700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 0x7f08f00a3750 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f08f8009bf0 tx=0x7f08f8009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55350/0 (socket says 172.21.15.90:55350) 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 -- 172.21.15.90:0/3458190846 learned_addr learned my addr 172.21.15.90:0/3458190846 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0907fff700 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2951d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f09077fe700 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb288780 0x5642cb288b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 -- 172.21.15.90:0/3458190846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb288780 msgr2=0x5642cb288b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb288780 0x5642cb288b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 -- 172.21.15.90:0/3458190846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 msgr2=0x5642cb2951d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2951d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 -- 172.21.15.90:0/3458190846 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5642cb05b3b0 con 0x5642cb2890a0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.982+0000 7f0906ffd700 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f08fc095df0 tx=0x7f08fc004860 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ca5853c4b0692458 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.984 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f08edffb700 1 -- 172.21.15.90:0/3458190846 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc0a1d70 con 0x5642cb2890a0 2024-04-06T14:16:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f08edffb700 1 -- 172.21.15.90:0/3458190846 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f08fc099b50 con 0x5642cb2890a0 2024-04-06T14:16:36.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f08edffb700 1 -- 172.21.15.90:0/3458190846 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc0a1b20 con 0x5642cb2890a0 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 -- 172.21.15.90:0/3458190846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 msgr2=0x5642cb2928a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f08fc095df0 tx=0x7f08fc004860 comp rx=0 tx=0).stop 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 -- 172.21.15.90:0/3458190846 shutdown_connections 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2951d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb288780 0x5642cb288b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 --2- 172.21.15.90:0/3458190846 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb2928a0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 -- 172.21.15.90:0/3458190846 >> 172.21.15.90:0/3458190846 conn(0x5642cb1e76a0 msgr2=0x5642cb283a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 -- 172.21.15.90:0/3458190846 shutdown_connections 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.983+0000 7f091a3bab80 1 -- 172.21.15.90:0/3458190846 wait complete. 2024-04-06T14:16:36.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 -- start start 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 Processor -- start 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb28fbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2902f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb175d80 con 0x5642cb292de0 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb175a20 con 0x5642cb288780 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb05d6a0 con 0x5642cb2890a0 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f091a3bab80 10 client.0 _ll_register_callbacks cb 0x5642cb17f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f09077fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb28fbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f09077fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55356/0 (socket says 172.21.15.90:55356) 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.984+0000 7f09077fe700 1 -- 172.21.15.90:0/2297087453 learned_addr learned my addr 172.21.15.90:0/2297087453 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:36.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 -- 172.21.15.90:0/2297087453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 msgr2=0x5642cb28fbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb28fbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f0907fff700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2902f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 -- 172.21.15.90:0/2297087453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 msgr2=0x5642cb2902f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2902f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 -- 172.21.15.90:0/2297087453 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f090819fc70 con 0x5642cb288780 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f0907fff700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2902f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f09077fe700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f08f8012870 tx=0x7f08f80047c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3b4e5540ba45a7ec server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f08e6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08f8004220 con 0x5642cb288780 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f08f8018070 con 0x5642cb288780 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5642cb19a380 con 0x5642cb288780 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f08f8004380 con 0x5642cb288780 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f091a3bab80 10 client.27023 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:36.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5642cb007b60 con 0x5642cb288780 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.985+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5642cb19a380 con 0x5642cb288780 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f08f8020900 con 0x5642cb288780 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 10 client.27023 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 10 client.27023 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 20 client.27023 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2a2b80 0x5642cb2a4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 4 client.27023 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5642cb1d2700 con 0x5642cb2a2b80 2024-04-06T14:16:36.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.986+0000 7f0906ffd700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2a2b80 0x5642cb2a4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:36.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.987+0000 7f0906ffd700 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2a2b80 0x5642cb2a4fa0 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8c322186ada7dfba server_cookie=23530c16b531a989 in_seq=0 out_seq=0 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.987+0000 7f08e6ffd700 10 client.27023 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2297087453 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5642cb1d2700 con 0x5642cb2a2b80 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f08e6ffd700 10 client.27023 handle_command_reply: tid=0 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 client.27023 shutdown 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 msgr2=0x5642cb292870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f08f8012870 tx=0x7f08f80047c0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 shutdown_connections 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb2902f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb292870 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2a2b80 0x5642cb2a4fa0 crc :-1 s=READY pgs=195 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2297087453 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb28fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 >> 172.21.15.90:0/2297087453 conn(0x5642cb1e76a0 msgr2=0x5642cb282c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:36.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f08e6ffd700 0 client.27023 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 shutdown_connections 2024-04-06T14:16:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.988+0000 7f091a3bab80 1 -- 172.21.15.90:0/2297087453 wait complete. 2024-04-06T14:16:36.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:36.989+0000 7f091a3bab80 20 client.27023 trim_cache size 0 max 16384 2024-04-06T14:16:37.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 Processor -- start 2024-04-06T14:16:37.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 -- start start 2024-04-06T14:16:37.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 0x5642cb2951d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb28f440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb292de0 0x5642cb299ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb28f9c0 con 0x5642cb288780 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb29a000 con 0x5642cb2890a0 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f091a3bab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb28e140 con 0x5642cb292de0 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f09077fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 0x5642cb2951d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f09077fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 0x5642cb2951d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54696/0 (socket says 172.21.15.90:54696) 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f09077fe700 1 -- 172.21.15.90:0/1123182492 learned_addr learned my addr 172.21.15.90:0/1123182492 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.004+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb28f440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0907fff700 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb292de0 0x5642cb299ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 -- 172.21.15.90:0/1123182492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb292de0 msgr2=0x5642cb299ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:37.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb292de0 0x5642cb299ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 -- 172.21.15.90:0/1123182492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 msgr2=0x5642cb2951d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 0x5642cb2951d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 -- 172.21.15.90:0/1123182492 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5642cb19fea0 con 0x5642cb2890a0 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f0906ffd700 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb28f440 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x5642cb289570 tx=0x7f08fc000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b7a8311aa7c1cc62 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.005+0000 7f08ee7fc700 1 -- 172.21.15.90:0/1123182492 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc0b2260 con 0x5642cb2890a0 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.006+0000 7f08ee7fc700 1 -- 172.21.15.90:0/1123182492 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f08fc0a1d70 con 0x5642cb2890a0 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.006+0000 7f08ee7fc700 1 -- 172.21.15.90:0/1123182492 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc0a1b40 con 0x5642cb2890a0 2024-04-06T14:16:37.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- 172.21.15.90:0/1123182492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 msgr2=0x5642cb28f440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb28f440 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x5642cb289570 tx=0x7f08fc000bc0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- 172.21.15.90:0/1123182492 shutdown_connections 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb292de0 0x5642cb299ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb288780 0x5642cb2951d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 --2- 172.21.15.90:0/1123182492 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb2890a0 0x5642cb28f440 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- 172.21.15.90:0/1123182492 >> 172.21.15.90:0/1123182492 conn(0x5642cb1e76a0 msgr2=0x5642cb2a98c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:37.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- 172.21.15.90:0/1123182492 shutdown_connections 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- 172.21.15.90:0/1123182492 wait complete. 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.007+0000 7f091a3bab80 1 -- start start 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 Processor -- start 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb28d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb297aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb297fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb298710 con 0x5642cb292de0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb298880 con 0x5642cb288780 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642cb2f5970 con 0x5642cb2890a0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f091a3bab80 10 client.0 _ll_register_callbacks cb 0x5642cb17f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f09077fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb28d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f0906ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb297aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f09077fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb28d7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55380/0 (socket says 172.21.15.90:55380) 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f09077fe700 1 -- 172.21.15.90:0/2567160176 learned_addr learned my addr 172.21.15.90:0/2567160176 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.008+0000 7f0907fff700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb297fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 -- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 msgr2=0x5642cb28d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb28d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 -- 172.21.15.90:0/2567160176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 msgr2=0x5642cb297fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb297fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 -- 172.21.15.90:0/2567160176 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5642cb05b3b0 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f0906ffd700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb297aa0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f08fc0993c0 tx=0x7f08fc09d7e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=282b66c1d22164ef server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f08e6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f08fc0b6070 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f08fc099cc0 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f08fc099b50 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f08fc098de0 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f091a3bab80 10 client.26809 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5642cb05fcd0 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.009+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f08fc099b50 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f08fc0b90b0 con 0x5642cb2890a0 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 10 client.26809 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 10 client.26809 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:37.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 20 client.26809 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2f6be0 0x5642cb2f8fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 4 client.26809 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5642cb028cf0 con 0x5642cb2f6be0 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f09077fe700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2f6be0 0x5642cb2f8fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f08e6ffd700 10 client.26809 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.010+0000 7f09077fe700 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2f6be0 0x5642cb2f8fd0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e916bc2302dc0c58 server_cookie=cd0283658bfa4e9e in_seq=0 out_seq=0 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f08e6ffd700 1 -- 172.21.15.90:0/2567160176 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5642cb028cf0 con 0x5642cb2f6be0 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f08e6ffd700 10 client.26809 handle_command_reply: tid=0 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 client.26809 shutdown 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 msgr2=0x5642cb297aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb297aa0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f08fc0993c0 tx=0x7f08fc09d7e0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 shutdown_connections 2024-04-06T14:16:37.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5642cb292de0 0x5642cb297fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5642cb288780 0x5642cb28d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5642cb2f6be0 0x5642cb2f8fd0 crc :-1 s=READY pgs=196 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 --2- 172.21.15.90:0/2567160176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5642cb2890a0 0x5642cb297aa0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f08e6ffd700 0 client.26809 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 >> 172.21.15.90:0/2567160176 conn(0x5642cb1e76a0 msgr2=0x5642cb28e4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 shutdown_connections 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 1 -- 172.21.15.90:0/2567160176 wait complete. 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.011+0000 7f091a3bab80 20 client.26809 trim_cache size 0 max 16384 2024-04-06T14:16:37.018 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 msgr2=0x7f08f00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 0x7f08f00a3750 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f08f8009bf0 tx=0x7f08f8009260 comp rx=0 tx=0).stop 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 msgr2=0x7f090810d970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f08fc00b500 tx=0x7f08fc00b8c0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 shutdown_connections 2024-04-06T14:16:37.019 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08f00a1290 0x7f08f00a3750 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f090819f180 0x7f090810deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f09080ffe70 0x7f090810d430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 --2- 172.21.15.90:0/1575150083 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0908104f70 0x7f090810d970 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 >> 172.21.15.90:0/1575150083 conn(0x7f0908076d50 msgr2=0x7f09080fdda0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 shutdown_connections 2024-04-06T14:16:37.020 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:37.012+0000 7f090e206700 1 -- 172.21.15.90:0/1575150083 wait complete. 2024-04-06T14:16:37.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:37 smithi090 ceph-mon[43420]: pgmap v349: 97 pgs: 97 active+clean; 6.5 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 25 MiB/s wr, 1.67k op/s 2024-04-06T14:16:37.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:37 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2227196109' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:37 smithi134 ceph-mon[47549]: pgmap v349: 97 pgs: 97 active+clean; 6.5 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 25 MiB/s wr, 1.67k op/s 2024-04-06T14:16:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:37 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2227196109' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:37.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:37 smithi105 ceph-mon[46969]: pgmap v349: 97 pgs: 97 active+clean; 6.5 GiB data, 18 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 25 MiB/s wr, 1.67k op/s 2024-04-06T14:16:37.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:37 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2227196109' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:16:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 650.0 s, 1629.8 tps, lat 63.153 ms stddev 79.850 2024-04-06T14:16:38.030 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:16:38.030 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:16:38.030 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 -- 172.21.15.90:0/2181985069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c10cf30 msgr2=0x7f913c114ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 --2- 172.21.15.90:0/2181985069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c10cf30 0x7f913c114ab0 secure :-1 s=READY pgs=756 cs=0 l=1 rev1=1 crypto rx=0x7f912c00aad0 tx=0x7f912c00ade0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 -- 172.21.15.90:0/2181985069 shutdown_connections 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 --2- 172.21.15.90:0/2181985069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c10cf30 0x7f913c114ab0 unknown :-1 s=CLOSED pgs=756 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 --2- 172.21.15.90:0/2181985069 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1088b0 0x7f913c108c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 --2- 172.21.15.90:0/2181985069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c109260 0x7f913c10c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 -- 172.21.15.90:0/2181985069 >> 172.21.15.90:0/2181985069 conn(0x7f913c06be00 msgr2=0x7f913c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.199 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 -- 172.21.15.90:0/2181985069 shutdown_connections 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.196+0000 7f914232c700 1 -- 172.21.15.90:0/2181985069 wait complete. 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 Processor -- start 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 -- start start 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c10cf30 0x7f913c1a6520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 0x7f913c1aad40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f913c1ab340 con 0x7f913c1088b0 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f913c1ab480 con 0x7f913c1aa8c0 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f914232c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f913c1ab5f0 con 0x7f913c10cf30 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f913bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f913bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54716/0 (socket says 172.21.15.90:54716) 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f913b7fe700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c10cf30 0x7f913c1a6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f913bfff700 1 -- 172.21.15.90:0/1139118933 learned_addr learned my addr 172.21.15.90:0/1139118933 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.200 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.197+0000 7f91408c9700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 0x7f913c1aad40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 -- 172.21.15.90:0/1139118933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c10cf30 msgr2=0x7f913c1a6520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c10cf30 0x7f913c1a6520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 -- 172.21.15.90:0/1139118933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 msgr2=0x7f913c1aad40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 0x7f913c1aad40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 -- 172.21.15.90:0/1139118933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f912c00a700 con 0x7f913c1088b0 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f91408c9700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 0x7f913c1aad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f913bfff700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f913400ef10 tx=0x7f913400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f91397fa700 1 -- 172.21.15.90:0/1139118933 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f913400c9c0 con 0x7f913c1088b0 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f913c1166b0 con 0x7f913c1088b0 2024-04-06T14:16:38.201 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f91397fa700 1 -- 172.21.15.90:0/1139118933 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f913400f040 con 0x7f913c1088b0 2024-04-06T14:16:38.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f91397fa700 1 -- 172.21.15.90:0/1139118933 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9134014480 con 0x7f913c1088b0 2024-04-06T14:16:38.202 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.198+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f913c116bb0 con 0x7f913c1088b0 2024-04-06T14:16:38.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f91397fa700 1 -- 172.21.15.90:0/1139118933 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9134007690 con 0x7f913c1088b0 2024-04-06T14:16:38.203 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f914e4e0b80 1 Processor -- start 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f91397fa700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 0x7f91200a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f914e4e0b80 1 -- start start 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f91397fa700 1 -- 172.21.15.90:0/1139118933 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f91340ccf80 con 0x7f913c1088b0 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f91408c9700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 0x7f91200a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.203+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d39920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d3c250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c87a10 con 0x5650a7d30120 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c853d0 con 0x5650a7d39e60 2024-04-06T14:16:38.205 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c80cc0 con 0x5650a7d2f800 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d39920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38124/0 (socket says 172.21.15.90:38124) 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 -- 172.21.15.90:0/1042005203 learned_addr learned my addr 172.21.15.90:0/1042005203 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f91408c9700 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d3c250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 -- 172.21.15.90:0/1042005203 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 msgr2=0x5650a7d3c250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.206 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d3c250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 -- 172.21.15.90:0/1042005203 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 msgr2=0x5650a7d39920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d39920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.204+0000 7f913bfff700 1 -- 172.21.15.90:0/1042005203 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5650a7b02430 con 0x5650a7d2f800 2024-04-06T14:16:38.207 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.205+0000 7f913bfff700 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f9134096a80 tx=0x7f9134099eb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7f4f7950f80f4f3f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.205+0000 7f91327fc700 1 -- 172.21.15.90:0/1042005203 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f913409a710 con 0x5650a7d2f800 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.205+0000 7f91408c9700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 0x7f91200a3830 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f912c00c690 tx=0x7f912c003ad0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.205+0000 7f91327fc700 1 -- 172.21.15.90:0/1042005203 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f913409a010 con 0x5650a7d2f800 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.205+0000 7f91327fc700 1 -- 172.21.15.90:0/1042005203 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9134099500 con 0x5650a7d2f800 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1042005203 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 msgr2=0x5650a7d2fbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f9134096a80 tx=0x7f9134099eb0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1042005203 shutdown_connections 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d3c250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d2fbe0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1042005203 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d39920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1042005203 >> 172.21.15.90:0/1042005203 conn(0x5650a7c8e720 msgr2=0x5650a7d2ab00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1042005203 shutdown_connections 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1042005203 wait complete. 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.206+0000 7f914e4e0b80 1 -- start start 2024-04-06T14:16:38.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.207+0000 7f914e4e0b80 1 Processor -- start 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d36c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d39e60 0x5650a7d37370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c1ce00 con 0x5650a7d30120 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c1caa0 con 0x5650a7d2f800 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7b04720 con 0x5650a7d39e60 2024-04-06T14:16:38.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.209+0000 7f914e4e0b80 10 client.0 _ll_register_callbacks cb 0x5650a7c264f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:38.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55422/0 (socket says 172.21.15.90:55422) 2024-04-06T14:16:38.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 -- 172.21.15.90:0/1878833525 learned_addr learned my addr 172.21.15.90:0/1878833525 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 -- 172.21.15.90:0/1878833525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d39e60 msgr2=0x5650a7d37370 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d39e60 0x5650a7d37370 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 -- 172.21.15.90:0/1878833525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 msgr2=0x5650a7d36c30 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d36c30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.210+0000 7f913bfff700 1 -- 172.21.15.90:0/1878833525 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f913c116bb0 con 0x5650a7d2f800 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f913bfff700 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f9134005bb0 tx=0x7f913409a710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b93d1814cefe8045 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f911f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:38.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91340a2490 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f913409a480 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5650a7c41400 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f91340a2000 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f914e4e0b80 10 client.27041 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.211+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5650a7aaebe0 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5650a7c41400 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f91340b8160 con 0x5650a7d2f800 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 10 client.27041 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 10 client.27041 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 20 client.27041 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:38.213 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d49620 0x5650a7d4ba40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 4 client.27041 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:38.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.212+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5650a7c79780 con 0x5650a7d49620 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.213+0000 7f913b7fe700 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d49620 0x5650a7d4ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f913b7fe700 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d49620 0x5650a7d4ba40 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1390b14bb28d79b1 server_cookie=f376843f05f24314 in_seq=0 out_seq=0 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f911f7fe700 10 client.27041 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f911f7fe700 1 -- 172.21.15.90:0/1878833525 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5650a7c79780 con 0x5650a7d49620 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f911f7fe700 10 client.27041 handle_command_reply: tid=0 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f914e4e0b80 1 client.27041 shutdown 2024-04-06T14:16:38.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 msgr2=0x5650a7d398a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.214+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f9134005bb0 tx=0x7f913409a710 comp rx=0 tx=0).stop 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 shutdown_connections 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d49620 0x5650a7d4ba40 crc :-1 s=READY pgs=197 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d39e60 0x5650a7d37370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f911f7fe700 0 client.27041 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d398a0 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/1878833525 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d36c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 >> 172.21.15.90:0/1878833525 conn(0x5650a7c8e720 msgr2=0x5650a7d29ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 shutdown_connections 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 1 -- 172.21.15.90:0/1878833525 wait complete. 2024-04-06T14:16:38.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.215+0000 7f914e4e0b80 20 client.27041 trim_cache size 0 max 16384 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.237+0000 7f914e4e0b80 1 Processor -- start 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.237+0000 7f914e4e0b80 1 -- start start 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.237+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d30120 0x5650a7d2a5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d39e60 0x5650a7d2aae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7c41400 con 0x5650a7d39e60 2024-04-06T14:16:38.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7d46600 con 0x5650a7d2f800 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7d41020 con 0x5650a7d30120 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f91408c9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d39e60 0x5650a7d2aae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913b7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d30120 0x5650a7d2a5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55424/0 (socket says 172.21.15.90:55424) 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 -- 172.21.15.90:0/3740141389 learned_addr learned my addr 172.21.15.90:0/3740141389 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 -- 172.21.15.90:0/3740141389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d30120 msgr2=0x5650a7d2a5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d30120 0x5650a7d2a5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 -- 172.21.15.90:0/3740141389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d39e60 msgr2=0x5650a7d2aae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d39e60 0x5650a7d2aae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 -- 172.21.15.90:0/3740141389 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5650a7d50690 con 0x5650a7d2f800 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f913bfff700 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f9134096510 tx=0x7f9134009eb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b3656642bea69d93 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.240 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f9132ffd700 1 -- 172.21.15.90:0/3740141389 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9134098670 con 0x5650a7d2f800 2024-04-06T14:16:38.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f9132ffd700 1 -- 172.21.15.90:0/3740141389 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f913407b030 con 0x5650a7d2f800 2024-04-06T14:16:38.241 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.238+0000 7f9132ffd700 1 -- 172.21.15.90:0/3740141389 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91340a2000 con 0x5650a7d2f800 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.239+0000 7f914e4e0b80 1 -- 172.21.15.90:0/3740141389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 msgr2=0x5650a7d40ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.239+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f9134096510 tx=0x7f9134009eb0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.240+0000 7f914e4e0b80 1 -- 172.21.15.90:0/3740141389 shutdown_connections 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.240+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d39e60 0x5650a7d2aae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.240+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d2f800 0x5650a7d40ae0 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.240+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/3740141389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d30120 0x5650a7d2a5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.240+0000 7f914e4e0b80 1 -- 172.21.15.90:0/3740141389 >> 172.21.15.90:0/3740141389 conn(0x5650a7c8e720 msgr2=0x5650a7d35500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.241+0000 7f914e4e0b80 1 -- 172.21.15.90:0/3740141389 shutdown_connections 2024-04-06T14:16:38.242 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.241+0000 7f914e4e0b80 1 -- 172.21.15.90:0/3740141389 wait complete. 2024-04-06T14:16:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.241+0000 7f914e4e0b80 1 -- start start 2024-04-06T14:16:38.243 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.241+0000 7f914e4e0b80 1 Processor -- start 2024-04-06T14:16:38.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d3f3f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.244 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d9d480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7d400e0 con 0x5650a7d30120 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7d40250 con 0x5650a7d39e60 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5650a7d9d9f0 con 0x5650a7d2f800 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f914e4e0b80 10 client.0 _ll_register_callbacks cb 0x5650a7c264f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f913b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f913b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54746/0 (socket says 172.21.15.90:54746) 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.242+0000 7f913b7fe700 1 -- 172.21.15.90:0/479018212 learned_addr learned my addr 172.21.15.90:0/479018212 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f91408c9700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d9d480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 -- 172.21.15.90:0/479018212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 msgr2=0x5650a7d3f3f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913bfff700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d3f3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.245 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d3f3f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 -- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 msgr2=0x5650a7d9d480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d9d480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 -- 172.21.15.90:0/479018212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5650a7aaebe0 con 0x5650a7d30120 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f91408c9700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d9d480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913bfff700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d3f3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f913b7fe700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f913c1a7530 tx=0x7f912801a520 comp rx=0 tx=0).ready entity=mon.0 client_cookie=344a985849860c05 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f911f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9128016040 con 0x5650a7d30120 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f912801aa90 con 0x5650a7d30120 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.243+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f913409a010 con 0x5650a7d30120 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f912801a730 con 0x5650a7d30120 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 10 client.17334 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:38.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5650a7d26be0 con 0x5650a7d30120 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f912802f080 con 0x5650a7d30120 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9128023440 con 0x5650a7d30120 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 10 client.17334 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 10 client.17334 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 20 client.17334 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d9f310 0x5650a7da1730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 4 client.17334 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.244+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x5650a7acfd70 con 0x5650a7d9f310 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.245+0000 7f913bfff700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d9f310 0x5650a7da1730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.246+0000 7f913bfff700 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d9f310 0x5650a7da1730 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b5f9e01877ea186f server_cookie=3541e3791c7365e7 in_seq=0 out_seq=0 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.246+0000 7f911f7fe700 10 client.17334 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.247+0000 7f911f7fe700 1 -- 172.21.15.90:0/479018212 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x5650a7acfd70 con 0x5650a7d9f310 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.247+0000 7f911f7fe700 10 client.17334 handle_command_reply: tid=0 2024-04-06T14:16:38.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.247+0000 7f914e4e0b80 1 client.17334 shutdown 2024-04-06T14:16:38.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.247+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 msgr2=0x5650a7d3f930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.247+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 secure :-1 s=READY pgs=758 cs=0 l=1 rev1=1 crypto rx=0x7f913c1a7530 tx=0x7f912801a520 comp rx=0 tx=0).stop 2024-04-06T14:16:38.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 shutdown_connections 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5650a7d39e60 0x5650a7d9d480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5650a7d9f310 0x5650a7da1730 crc :-1 s=READY pgs=198 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5650a7d2f800 0x5650a7d3f3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 --2- 172.21.15.90:0/479018212 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5650a7d30120 0x5650a7d3f930 unknown :-1 s=CLOSED pgs=758 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 >> 172.21.15.90:0/479018212 conn(0x5650a7c8e720 msgr2=0x5650a7d35140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f911f7fe700 0 client.17334 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 shutdown_connections 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 1 -- 172.21.15.90:0/479018212 wait complete. 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.248+0000 7f914e4e0b80 20 client.17334 trim_cache size 0 max 16384 2024-04-06T14:16:38.250 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:38.251 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:16:38.251 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "03d2f42d-13f0-44f1-9dd5-912f12a9a2f8", 2024-04-06T14:16:38.251 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:16:38.251 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:38.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 msgr2=0x7f91200a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 0x7f91200a3830 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f912c00c690 tx=0x7f912c003ad0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 msgr2=0x7f913c1a5fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 secure :-1 s=READY pgs=757 cs=0 l=1 rev1=1 crypto rx=0x7f913400ef10 tx=0x7f913400c3a0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 shutdown_connections 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91200a1370 0x7f91200a3830 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f913c1aa8c0 0x7f913c1aad40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f913c1088b0 0x7f913c1a5fe0 unknown :-1 s=CLOSED pgs=757 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 --2- 172.21.15.90:0/1139118933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f913c10cf30 0x7f913c1a6520 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 >> 172.21.15.90:0/1139118933 conn(0x7f913c06be00 msgr2=0x7f913c070ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 shutdown_connections 2024-04-06T14:16:38.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.249+0000 7f914232c700 1 -- 172.21.15.90:0/1139118933 wait complete. 2024-04-06T14:16:38.266 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:16:38.266 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:16:38.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 -- 172.21.15.90:0/333823527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a600ffcc0 msgr2=0x7f8a60103960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 --2- 172.21.15.90:0/333823527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a600ffcc0 0x7f8a60103960 secure :-1 s=READY pgs=759 cs=0 l=1 rev1=1 crypto rx=0x7f8a48009990 tx=0x7f8a48009ca0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 -- 172.21.15.90:0/333823527 shutdown_connections 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 --2- 172.21.15.90:0/333823527 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a600ffcc0 0x7f8a60103960 unknown :-1 s=CLOSED pgs=759 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 --2- 172.21.15.90:0/333823527 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a60105e60 0x7f8a60106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 --2- 172.21.15.90:0/333823527 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a600ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.474+0000 7f8a66e87700 1 -- 172.21.15.90:0/333823527 >> 172.21.15.90:0/333823527 conn(0x7f8a600761b0 msgr2=0x7f8a600fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.475+0000 7f8a66e87700 1 -- 172.21.15.90:0/333823527 shutdown_connections 2024-04-06T14:16:38.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.475+0000 7f8a66e87700 1 -- 172.21.15.90:0/333823527 wait complete. 2024-04-06T14:16:38.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.475+0000 7f8a66e87700 1 Processor -- start 2024-04-06T14:16:38.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 -- start start 2024-04-06T14:16:38.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a601a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 0x7f8a601a3c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a60105e60 0x7f8a6019d800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a6019de70 con 0x7f8a60105e60 2024-04-06T14:16:38.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a6019dfe0 con 0x7f8a600ffcc0 2024-04-06T14:16:38.477 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.476+0000 7f8a66e87700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8a6019e1a0 con 0x7f8a600ff220 2024-04-06T14:16:38.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 0x7f8a601a3c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 0x7f8a601a3c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55450/0 (socket says 172.21.15.90:55450) 2024-04-06T14:16:38.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a5ffff700 1 -- 172.21.15.90:0/3360063689 learned_addr learned my addr 172.21.15.90:0/3360063689 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.478 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a601a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a65424700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a60105e60 0x7f8a6019d800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 -- 172.21.15.90:0/3360063689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 msgr2=0x7f8a601a3c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 0x7f8a601a3c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 -- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a60105e60 msgr2=0x7f8a6019d800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a60105e60 0x7f8a6019d800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 -- 172.21.15.90:0/3360063689 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8a480095f0 con 0x7f8a600ff220 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.477+0000 7f8a64c23700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a601a36d0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f8a5000e9f0 tx=0x7f8a5000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.478+0000 7f8a5dffb700 1 -- 172.21.15.90:0/3360063689 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5000eee0 con 0x7f8a600ff220 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.478+0000 7f8a5dffb700 1 -- 172.21.15.90:0/3360063689 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a5000cbd0 con 0x7f8a600ff220 2024-04-06T14:16:38.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.478+0000 7f8a5dffb700 1 -- 172.21.15.90:0/3360063689 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a50018680 con 0x7f8a600ff220 2024-04-06T14:16:38.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.479+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8a6019e480 con 0x7f8a600ff220 2024-04-06T14:16:38.480 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.479+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a601aa6c0 con 0x7f8a600ff220 2024-04-06T14:16:38.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.482+0000 7f8a5dffb700 1 -- 172.21.15.90:0/3360063689 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8a5000cd40 con 0x7f8a600ff220 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 Processor -- start 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 -- start start 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34a1b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34ab8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d33f9990 con 0x5640d34a20a0 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d33f7350 con 0x5640d34abde0 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d33f2c40 con 0x5640d34a1780 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a5dffb700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 0x7f8a4c0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55452/0 (socket says 172.21.15.90:55452) 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a5ffff700 1 -- 172.21.15.90:0/1606478241 learned_addr learned my addr 172.21.15.90:0/1606478241 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a65424700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34a1b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.484+0000 7f8a5dffb700 1 -- 172.21.15.90:0/3360063689 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8a50014070 con 0x7f8a600ff220 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 -- 172.21.15.90:0/1606478241 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 msgr2=0x5640d34a1b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34a1b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 -- 172.21.15.90:0/1606478241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 msgr2=0x5640d34ab8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a64c23700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34ab8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a65424700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 0x7f8a4c0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34ab8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 -- 172.21.15.90:0/1606478241 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5640d32743b0 con 0x5640d34abde0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a64c23700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34ab8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e630 tx=0x7f8a58009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=44ae1176ed41fccc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a65424700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 0x7f8a4c0a3890 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f8a48005b40 tx=0x7f8a48009ca0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a56ffd700 1 -- 172.21.15.90:0/1606478241 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5800e040 con 0x5640d34abde0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a56ffd700 1 -- 172.21.15.90:0/1606478241 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a5800f040 con 0x5640d34abde0 2024-04-06T14:16:38.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.485+0000 7f8a56ffd700 1 -- 172.21.15.90:0/1606478241 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a58007500 con 0x5640d34abde0 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.486+0000 7f8a7303bb80 1 -- 172.21.15.90:0/1606478241 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 msgr2=0x5640d34ae1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.486+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e630 tx=0x7f8a58009920 comp rx=0 tx=0).stop 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 -- 172.21.15.90:0/1606478241 shutdown_connections 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34ae1d0 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34a1b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/1606478241 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34ab8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 -- 172.21.15.90:0/1606478241 >> 172.21.15.90:0/1606478241 conn(0x5640d34006a0 msgr2=0x5640d349ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 -- 172.21.15.90:0/1606478241 shutdown_connections 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 -- 172.21.15.90:0/1606478241 wait complete. 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 -- start start 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.487+0000 7f8a7303bb80 1 Processor -- start 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ab850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34a8be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34a92d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d338ed80 con 0x5640d34a20a0 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d338ea20 con 0x5640d34abde0 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d32766a0 con 0x5640d34a1780 2024-04-06T14:16:38.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a7303bb80 10 client.0 _ll_register_callbacks cb 0x5640d3398470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a64c23700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ab850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a64c23700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ab850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:38198/0 (socket says 172.21.15.90:38198) 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a64c23700 1 -- 172.21.15.90:0/717392722 learned_addr learned my addr 172.21.15.90:0/717392722 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.488+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34a8be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a65424700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34a92d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 -- 172.21.15.90:0/717392722 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 msgr2=0x5640d34ab850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ab850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 -- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 msgr2=0x5640d34a92d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34a92d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 -- 172.21.15.90:0/717392722 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a5800e2e0 con 0x5640d34a20a0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.489+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34a8be0 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e600 tx=0x7f8a58004900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=48bb0480b95e5fd8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.490+0000 7f8a3ffff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.490+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5800e040 con 0x5640d34a20a0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.490+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8a580093a0 con 0x5640d34a20a0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.491+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5640d33b3380 con 0x5640d34a20a0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.491+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a58007500 con 0x5640d34a20a0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.491+0000 7f8a7303bb80 10 client.17343 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.491+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5640d3220b60 con 0x5640d34a20a0 2024-04-06T14:16:38.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5640d33b3380 con 0x5640d34a20a0 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a58015610 con 0x5640d34a20a0 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 10 client.17343 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 10 client.17343 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 20 client.17343 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d34cc6f0 0x5640d34ceb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 4 client.17343 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.492+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5640d33eb700 con 0x5640d34cc6f0 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.493+0000 7f8a64c23700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d34cc6f0 0x5640d34ceb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.493+0000 7f8a64c23700 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d34cc6f0 0x5640d34ceb10 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=31db11c4836b66bd server_cookie=ae23a11e91c1ee21 in_seq=0 out_seq=0 2024-04-06T14:16:38.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.494+0000 7f8a3ffff700 10 client.17343 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.494+0000 7f8a3ffff700 1 -- 172.21.15.90:0/717392722 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5640d33eb700 con 0x5640d34cc6f0 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.494+0000 7f8a3ffff700 10 client.17343 handle_command_reply: tid=0 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.494+0000 7f8a7303bb80 1 client.17343 shutdown 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 msgr2=0x5640d34a8be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34a8be0 secure :-1 s=READY pgs=760 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e600 tx=0x7f8a58004900 comp rx=0 tx=0).stop 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 shutdown_connections 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34abde0 0x5640d34a92d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d34cc6f0 0x5640d34ceb10 crc :-1 s=READY pgs=199 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ab850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/717392722 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34a20a0 0x5640d34a8be0 unknown :-1 s=CLOSED pgs=760 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 >> 172.21.15.90:0/717392722 conn(0x5640d34006a0 msgr2=0x5640d349bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a3ffff700 0 client.17343 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 shutdown_connections 2024-04-06T14:16:38.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 1 -- 172.21.15.90:0/717392722 wait complete. 2024-04-06T14:16:38.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.495+0000 7f8a7303bb80 20 client.17343 trim_cache size 0 max 16384 2024-04-06T14:16:38.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.538+0000 7f8a7303bb80 1 Processor -- start 2024-04-06T14:16:38.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 -- start start 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ae1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b2a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d34c9510 con 0x5640d34abde0 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d3494330 con 0x5640d34a20a0 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d34b2f70 con 0x5640d34a1780 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a65424700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a65424700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54814/0 (socket says 172.21.15.90:54814) 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a65424700 1 -- 172.21.15.90:0/951191145 learned_addr learned my addr 172.21.15.90:0/951191145 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a64c23700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ae1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.547+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b2a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 -- 172.21.15.90:0/951191145 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 msgr2=0x5640d34ae1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ae1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 -- 172.21.15.90:0/951191145 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 msgr2=0x5640d34ab380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 -- 172.21.15.90:0/951191145 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5640d33b8ea0 con 0x5640d34a20a0 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a65424700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b2a30 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e5d0 tx=0x7f8a58005a30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=44ee750272cf6787 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a577fe700 1 -- 172.21.15.90:0/951191145 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5800ec20 con 0x5640d34a20a0 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a577fe700 1 -- 172.21.15.90:0/951191145 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a58015c90 con 0x5640d34a20a0 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.548+0000 7f8a577fe700 1 -- 172.21.15.90:0/951191145 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a5801ee30 con 0x5640d34a20a0 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 -- 172.21.15.90:0/951191145 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 msgr2=0x5640d34b2a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.550 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b2a30 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8a5800e5d0 tx=0x7f8a58005a30 comp rx=0 tx=0).stop 2024-04-06T14:16:38.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 -- 172.21.15.90:0/951191145 shutdown_connections 2024-04-06T14:16:38.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34ab380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34ae1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/951191145 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b2a30 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.549+0000 7f8a7303bb80 1 -- 172.21.15.90:0/951191145 >> 172.21.15.90:0/951191145 conn(0x5640d34006a0 msgr2=0x5640d34a3f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.550+0000 7f8a7303bb80 1 -- 172.21.15.90:0/951191145 shutdown_connections 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.550+0000 7f8a7303bb80 1 -- 172.21.15.90:0/951191145 wait complete. 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.550+0000 7f8a7303bb80 1 -- start start 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.550+0000 7f8a7303bb80 1 Processor -- start 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34b2470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34b0b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d34b1260 con 0x5640d34abde0 2024-04-06T14:16:38.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d34b13d0 con 0x5640d34a20a0 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5640d34b1590 con 0x5640d34a1780 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a7303bb80 10 client.0 _ll_register_callbacks cb 0x5640d3398470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.551+0000 7f8a64c23700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34b2470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55490/0 (socket says 172.21.15.90:55490) 2024-04-06T14:16:38.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 -- 172.21.15.90:0/66001508 learned_addr learned my addr 172.21.15.90:0/66001508 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 -- 172.21.15.90:0/66001508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 msgr2=0x5640d34b2470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34b2470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 -- 172.21.15.90:0/66001508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 msgr2=0x5640d34b0b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34b0b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 -- 172.21.15.90:0/66001508 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8a48020070 con 0x5640d34a20a0 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.552+0000 7f8a5ffff700 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f8a58003dc0 tx=0x7f8a580045d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6587519217dcfa97 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:16:38.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.553+0000 7f8a3ffff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.553+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8a58015650 con 0x5640d34a20a0 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.553+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8a58032470 con 0x5640d34a20a0 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.553+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8a5800f040 con 0x5640d34a20a0 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8a580157b0 con 0x5640d34a20a0 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 10 client.27062 fetch_fsmap learned FSMap version 24 2024-04-06T14:16:38.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5640d3220b60 con 0x5640d34a20a0 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8a5800f040 con 0x5640d34a20a0 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f8a58043070 con 0x5640d34a20a0 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 10 client.27062 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 10 client.27062 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 20 client.27062 populate_metadata read hostname 'smithi090' 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d3521360 0x5640d3523750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:16:38.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 4 client.27062 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.554+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5640d3241cf0 con 0x5640d3521360 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.555+0000 7f8a64c23700 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d3521360 0x5640d3523750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.555+0000 7f8a64c23700 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d3521360 0x5640d3523750 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1def516c9dd8b0de server_cookie=4a88deb19588d5c in_seq=0 out_seq=0 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.555+0000 7f8a3ffff700 10 client.27062 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.555+0000 7f8a3ffff700 1 -- 172.21.15.90:0/66001508 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x5640d3241cf0 con 0x5640d3521360 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.555+0000 7f8a3ffff700 10 client.27062 handle_command_reply: tid=0 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 client.27062 shutdown 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 msgr2=0x5640d34b29b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f8a58003dc0 tx=0x7f8a580045d0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 shutdown_connections 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5640d34abde0 0x5640d34b0b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5640d3521360 0x5640d3523750 crc :-1 s=READY pgs=200 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5640d34a1780 0x5640d34b2470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 --2- 172.21.15.90:0/66001508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5640d34a20a0 0x5640d34b29b0 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 >> 172.21.15.90:0/66001508 conn(0x5640d34006a0 msgr2=0x5640d34a7470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a3ffff700 0 client.27062 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 shutdown_connections 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.556+0000 7f8a7303bb80 1 -- 172.21.15.90:0/66001508 wait complete. 2024-04-06T14:16:38.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.557+0000 7f8a7303bb80 20 client.27062 trim_cache size 0 max 16384 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "03d2f42d-13f0-44f1-9dd5-912f12a9a2f8": { 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "03d2f42d-13f0-44f1-9dd5-912f12a9a2f8", 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:16:38.560 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:16:38.561 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:16:38.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 msgr2=0x7f8a4c0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 0x7f8a4c0a3890 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f8a48005b40 tx=0x7f8a48009ca0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 msgr2=0x7f8a601a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a601a36d0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f8a5000e9f0 tx=0x7f8a5000ed00 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 shutdown_connections 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8a4c0a13d0 0x7f8a4c0a3890 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8a60105e60 0x7f8a6019d800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8a600ff220 0x7f8a601a36d0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 --2- 172.21.15.90:0/3360063689 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8a600ffcc0 0x7f8a601a3c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 >> 172.21.15.90:0/3360063689 conn(0x7f8a600761b0 msgr2=0x7f8a60101450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 shutdown_connections 2024-04-06T14:16:38.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:16:38.559+0000 7f8a66e87700 1 -- 172.21.15.90:0/3360063689 wait complete. 2024-04-06T14:16:38.587 INFO:tasks.cephfs.filesystem:scrub status for tag:03d2f42d-13f0-44f1-9dd5-912f12a9a2f8 - {'path': '/', 'tag': '03d2f42d-13f0-44f1-9dd5-912f12a9a2f8', 'options': 'recursive,force'} 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: pgmap v350: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 27 MiB/s wr, 1.96k op/s 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:38 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:16:39.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: pgmap v350: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 27 MiB/s wr, 1.96k op/s 2024-04-06T14:16:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:16:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:16:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:38 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:16:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: pgmap v350: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 27 MiB/s wr, 1.96k op/s 2024-04-06T14:16:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:16:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:16:39.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:16:39.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:38 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:16:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:39 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:16:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:39 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:16:40.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:39 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:16:40.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:39 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:16:40.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:39 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:16:40.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:39 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:16:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:39 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:16:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:39 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:16:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:39 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:16:41.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:40 smithi134 ceph-mon[47549]: pgmap v351: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 24 MiB/s wr, 1.86k op/s 2024-04-06T14:16:41.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:40 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:16:41.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:40 smithi105 ceph-mon[46969]: pgmap v351: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 24 MiB/s wr, 1.86k op/s 2024-04-06T14:16:41.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:40 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:16:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:40 smithi090 ceph-mon[43420]: pgmap v351: 97 pgs: 97 active+clean; 6.5 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 70 KiB/s rd, 24 MiB/s wr, 1.86k op/s 2024-04-06T14:16:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:40 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:16:43.028 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 655.0 s, 1610.1 tps, lat 61.518 ms stddev 96.693 2024-04-06T14:16:43.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:42 smithi090 ceph-mon[43420]: pgmap v352: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 298 KiB/s rd, 31 MiB/s wr, 2.41k op/s 2024-04-06T14:16:43.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:42 smithi134 ceph-mon[47549]: pgmap v352: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 298 KiB/s rd, 31 MiB/s wr, 2.41k op/s 2024-04-06T14:16:43.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:42 smithi105 ceph-mon[46969]: pgmap v352: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 298 KiB/s rd, 31 MiB/s wr, 2.41k op/s 2024-04-06T14:16:45.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:45 smithi090 ceph-mon[43420]: pgmap v353: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 281 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:45.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:45 smithi134 ceph-mon[47549]: pgmap v353: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 281 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:45 smithi105 ceph-mon[46969]: pgmap v353: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 281 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:16:47.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:47 smithi090 ceph-mon[43420]: pgmap v354: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 19 MiB/s wr, 1.43k op/s 2024-04-06T14:16:47.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:47 smithi134 ceph-mon[47549]: pgmap v354: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 19 MiB/s wr, 1.43k op/s 2024-04-06T14:16:47.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:47 smithi105 ceph-mon[46969]: pgmap v354: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 19 MiB/s wr, 1.43k op/s 2024-04-06T14:16:48.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 660.0 s, 1631.2 tps, lat 61.741 ms stddev 74.220 2024-04-06T14:16:49.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:48 smithi105 ceph-mon[46969]: pgmap v355: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 302 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:16:49.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:48 smithi090 ceph-mon[43420]: pgmap v355: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 302 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:16:49.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:48 smithi134 ceph-mon[47549]: pgmap v355: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 302 KiB/s rd, 19 MiB/s wr, 1.41k op/s 2024-04-06T14:16:51.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:51 smithi090 ceph-mon[43420]: pgmap v356: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 263 KiB/s rd, 15 MiB/s wr, 1.07k op/s 2024-04-06T14:16:51.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:51 smithi134 ceph-mon[47549]: pgmap v356: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 263 KiB/s rd, 15 MiB/s wr, 1.07k op/s 2024-04-06T14:16:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:51 smithi105 ceph-mon[46969]: pgmap v356: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 263 KiB/s rd, 15 MiB/s wr, 1.07k op/s 2024-04-06T14:16:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 665.0 s, 1576.7 tps, lat 63.408 ms stddev 71.105 2024-04-06T14:16:53.299 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:53 smithi090 ceph-mon[43420]: pgmap v357: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 288 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:16:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:53 smithi134 ceph-mon[47549]: pgmap v357: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 288 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:16:53.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:53 smithi105 ceph-mon[46969]: pgmap v357: 97 pgs: 97 active+clean; 6.6 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 288 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:16:55.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:55 smithi090 ceph-mon[43420]: pgmap v358: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 678 op/s 2024-04-06T14:16:55.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:55 smithi134 ceph-mon[47549]: pgmap v358: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 678 op/s 2024-04-06T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:55 smithi105 ceph-mon[46969]: pgmap v358: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 678 op/s 2024-04-06T14:16:57.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:57 smithi090 ceph-mon[43420]: pgmap v359: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 106 KiB/s rd, 9.9 MiB/s wr, 642 op/s 2024-04-06T14:16:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:57 smithi134 ceph-mon[47549]: pgmap v359: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 106 KiB/s rd, 9.9 MiB/s wr, 642 op/s 2024-04-06T14:16:57.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:57 smithi105 ceph-mon[46969]: pgmap v359: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 106 KiB/s rd, 9.9 MiB/s wr, 642 op/s 2024-04-06T14:16:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 670.0 s, 1578.2 tps, lat 62.780 ms stddev 69.974 2024-04-06T14:16:59.143 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:16:58 smithi134 ceph-mon[47549]: pgmap v360: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 11 MiB/s wr, 668 op/s 2024-04-06T14:16:59.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:16:58 smithi105 ceph-mon[46969]: pgmap v360: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 11 MiB/s wr, 668 op/s 2024-04-06T14:16:59.433 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:16:58 smithi090 ceph-mon[43420]: pgmap v360: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 11 MiB/s wr, 668 op/s 2024-04-06T14:17:01.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:01 smithi090 ceph-mon[43420]: pgmap v361: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 8.9 MiB/s wr, 557 op/s 2024-04-06T14:17:01.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:01 smithi134 ceph-mon[47549]: pgmap v361: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 8.9 MiB/s wr, 557 op/s 2024-04-06T14:17:01.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:01 smithi105 ceph-mon[46969]: pgmap v361: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 8.9 MiB/s wr, 557 op/s 2024-04-06T14:17:03.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 675.0 s, 1246.0 tps, lat 78.551 ms stddev 131.599 2024-04-06T14:17:03.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:03 smithi090 ceph-mon[43420]: pgmap v362: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 129 KiB/s rd, 13 MiB/s wr, 822 op/s 2024-04-06T14:17:03.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:03 smithi134 ceph-mon[47549]: pgmap v362: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 129 KiB/s rd, 13 MiB/s wr, 822 op/s 2024-04-06T14:17:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:03 smithi105 ceph-mon[46969]: pgmap v362: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 129 KiB/s rd, 13 MiB/s wr, 822 op/s 2024-04-06T14:17:05.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:05 smithi134 ceph-mon[47549]: pgmap v363: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 13 MiB/s wr, 989 op/s 2024-04-06T14:17:05.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:05 smithi105 ceph-mon[46969]: pgmap v363: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 13 MiB/s wr, 989 op/s 2024-04-06T14:17:05.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:05 smithi090 ceph-mon[43420]: pgmap v363: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 104 KiB/s rd, 13 MiB/s wr, 989 op/s 2024-04-06T14:17:07.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:07 smithi134 ceph-mon[47549]: pgmap v364: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:17:07.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:07 smithi105 ceph-mon[46969]: pgmap v364: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:17:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:07 smithi090 ceph-mon[43420]: pgmap v364: 97 pgs: 97 active+clean; 6.7 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 114 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:17:08.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 680.0 s, 1450.9 tps, lat 70.376 ms stddev 81.135 2024-04-06T14:17:08.588 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:17:08.589 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:17:08.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 -- 172.21.15.90:0/747874643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 msgr2=0x7f609410cad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 --2- 172.21.15.90:0/747874643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f609410cad0 secure :-1 s=READY pgs=761 cs=0 l=1 rev1=1 crypto rx=0x7f608c00b0d0 tx=0x7f608c00b3e0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 -- 172.21.15.90:0/747874643 shutdown_connections 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 --2- 172.21.15.90:0/747874643 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f609410cad0 unknown :-1 s=CLOSED pgs=761 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 --2- 172.21.15.90:0/747874643 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 0x7f6094106180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 --2- 172.21.15.90:0/747874643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 -- 172.21.15.90:0/747874643 >> 172.21.15.90:0/747874643 conn(0x7f60940769b0 msgr2=0x7f6094076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 -- 172.21.15.90:0/747874643 shutdown_connections 2024-04-06T14:17:08.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.894+0000 7f6098c41700 1 -- 172.21.15.90:0/747874643 wait complete. 2024-04-06T14:17:08.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 Processor -- start 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 -- start start 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094195070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f60941955b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 0x7f6094199900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6094199f40 con 0x7f60941008c0 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f609419a0b0 con 0x7f60940fff00 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6098c41700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f609419a270 con 0x7f6094105da0 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6091d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f60941955b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094195070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6091d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f60941955b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47420/0 (socket says 172.21.15.90:47420) 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6091d9b700 1 -- 172.21.15.90:0/3833392641 learned_addr learned my addr 172.21.15.90:0/3833392641 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:08.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 0x7f6094199900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 -- 172.21.15.90:0/3833392641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 msgr2=0x7f6094199900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 0x7f6094199900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 -- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 msgr2=0x7f60941955b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f60941955b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.895+0000 7f609259c700 1 -- 172.21.15.90:0/3833392641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f608c009d00 con 0x7f60940fff00 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.896+0000 7f609259c700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094195070 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f6084012be0 tx=0x7f6084012ef0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.896+0000 7f60837fe700 1 -- 172.21.15.90:0/3833392641 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f608401c070 con 0x7f60940fff00 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.896+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f609419a4f0 con 0x7f60940fff00 2024-04-06T14:17:08.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.896+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60941a1d90 con 0x7f60940fff00 2024-04-06T14:17:08.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.896+0000 7f60837fe700 1 -- 172.21.15.90:0/3833392641 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6084013040 con 0x7f60940fff00 2024-04-06T14:17:08.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.900+0000 7f60837fe700 1 -- 172.21.15.90:0/3833392641 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6084003c70 con 0x7f60940fff00 2024-04-06T14:17:08.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.900+0000 7f60a4df5b80 1 Processor -- start 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 -- start start 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b29460ab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461f9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b294562990 con 0x55b29460b0a0 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b294560350 con 0x55b29461d5f0 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29455bc40 con 0x55b29460a780 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47430/0 (socket says 172.21.15.90:47430) 2024-04-06T14:17:08.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 -- 172.21.15.90:0/555530053 learned_addr learned my addr 172.21.15.90:0/555530053 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f6092d9d700 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b29460ab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 -- 172.21.15.90:0/555530053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 msgr2=0x55b29460ab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b29460ab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 -- 172.21.15.90:0/555530053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 msgr2=0x55b29461f9e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.901+0000 7f609259c700 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461f9e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.902+0000 7f609259c700 1 -- 172.21.15.90:0/555530053 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b2943dd3b0 con 0x55b29460b0a0 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.902+0000 7f6092d9d700 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b29460ab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:08.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.902+0000 7f609259c700 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f608401b000 tx=0x7f6084004ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44602b5ef9ff7332 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.903+0000 7f60837fe700 1 -- 172.21.15.90:0/3833392641 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f608404d070 con 0x7f60940fff00 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.903+0000 7f60837fe700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 0x7f607c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.903+0000 7f60837fe700 1 -- 172.21.15.90:0/3833392641 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f60840ce590 con 0x7f60940fff00 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f6080ff9700 1 -- 172.21.15.90:0/555530053 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60840ce880 con 0x55b29460b0a0 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f6080ff9700 1 -- 172.21.15.90:0/555530053 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f60840cf090 con 0x55b29460b0a0 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f6080ff9700 1 -- 172.21.15.90:0/555530053 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f60840d02d0 con 0x55b29460b0a0 2024-04-06T14:17:08.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 0x7f607c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f60a4df5b80 1 -- 172.21.15.90:0/555530053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 msgr2=0x55b29461d0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 secure :-1 s=READY pgs=762 cs=0 l=1 rev1=1 crypto rx=0x7f608401b000 tx=0x7f6084004ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 0x7f607c0a3750 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f608c009fd0 tx=0x7f608c00b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.904+0000 7f60a4df5b80 1 -- 172.21.15.90:0/555530053 shutdown_connections 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461f9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b29460ab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/555530053 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b29461d0b0 unknown :-1 s=CLOSED pgs=762 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 -- 172.21.15.90:0/555530053 >> 172.21.15.90:0/555530053 conn(0x55b2945696a0 msgr2=0x55b294605a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 -- 172.21.15.90:0/555530053 shutdown_connections 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.905+0000 7f60a4df5b80 1 -- 172.21.15.90:0/555530053 wait complete. 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 -- start start 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 Processor -- start 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460a780 0x55b294616b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29461d5f0 0x55b29461c870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b2944f7d80 con 0x55b29460b0a0 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b2944f7a20 con 0x55b29460a780 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b2943df6a0 con 0x55b29461d5f0 2024-04-06T14:17:08.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.906+0000 7f60a4df5b80 10 client.0 _ll_register_callbacks cb 0x55b294501470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:08.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47444/0 (socket says 172.21.15.90:47444) 2024-04-06T14:17:08.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 -- 172.21.15.90:0/375548176 learned_addr learned my addr 172.21.15.90:0/375548176 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:08.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6092d9d700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29461d5f0 0x55b29461c870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 -- 172.21.15.90:0/375548176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29461d5f0 msgr2=0x55b29461c870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29461d5f0 0x55b29461c870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 -- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460a780 msgr2=0x55b294616b50 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460a780 0x55b294616b50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 -- 172.21.15.90:0/375548176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60941a1d90 con 0x55b29460b0a0 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6091d9b700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f60880116c0 tx=0x7f60880119d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6a6b79644230f84f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6079ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:08.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6088004020 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.907+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6088004180 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b29451c380 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6088004340 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 10 client.17358 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b294389b60 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b29451c380 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6088020020 con 0x55b29460b0a0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 10 client.17358 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 10 client.17358 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.908+0000 7f60a4df5b80 20 client.17358 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b294635ad0 0x55b294637ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f60a4df5b80 4 client.17358 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b294554700 con 0x55b294635ad0 2024-04-06T14:17:08.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f609259c700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b294635ad0 0x55b294637ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f609259c700 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b294635ad0 0x55b294637ef0 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=28ab2f945ad529cb server_cookie=f918204dd9851231 in_seq=0 out_seq=0 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.909+0000 7f6079ffb700 10 client.17358 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f6079ffb700 1 -- 172.21.15.90:0/375548176 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b294554700 con 0x55b294635ad0 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f6079ffb700 10 client.17358 handle_command_reply: tid=0 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 client.17358 shutdown 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 msgr2=0x55b294617090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 secure :-1 s=READY pgs=763 cs=0 l=1 rev1=1 crypto rx=0x7f60880116c0 tx=0x7f60880119d0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 shutdown_connections 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b294635ad0 0x55b294637ef0 crc :-1 s=READY pgs=201 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29461d5f0 0x55b29461c870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460a780 0x55b294616b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/375548176 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294617090 unknown :-1 s=CLOSED pgs=763 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 >> 172.21.15.90:0/375548176 conn(0x55b2945696a0 msgr2=0x55b294604c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f6079ffb700 0 client.17358 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:08.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 shutdown_connections 2024-04-06T14:17:08.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.910+0000 7f60a4df5b80 1 -- 172.21.15.90:0/375548176 wait complete. 2024-04-06T14:17:08.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.911+0000 7f60a4df5b80 20 client.17358 trim_cache size 0 max 16384 2024-04-06T14:17:08.933 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 Processor -- start 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 -- start start 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294616770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460b0a0 0x55b294616cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 0x55b29460f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b2946132d0 con 0x55b29461d5f0 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29460c630 con 0x55b29460b0a0 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29461ce70 con 0x55b29460a780 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f609259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294616770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f609259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294616770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39896/0 (socket says 172.21.15.90:39896) 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f609259c700 1 -- 172.21.15.90:0/4182788197 learned_addr learned my addr 172.21.15.90:0/4182788197 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:08.934 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f6091d9b700 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460b0a0 0x55b294616cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.933+0000 7f6092d9d700 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 0x55b29460f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 -- 172.21.15.90:0/4182788197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 msgr2=0x55b294616770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294616770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 -- 172.21.15.90:0/4182788197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460b0a0 msgr2=0x55b294616cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460b0a0 0x55b294616cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 -- 172.21.15.90:0/4182788197 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b294521ea0 con 0x55b29461d5f0 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f6092d9d700 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 0x55b29460f1b0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f608c009cb0 tx=0x7f608c01f000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=87083bf13f0ffcb7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.935 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.934+0000 7f60817fa700 1 -- 172.21.15.90:0/4182788197 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f608c00e040 con 0x55b29461d5f0 2024-04-06T14:17:08.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.935+0000 7f60817fa700 1 -- 172.21.15.90:0/4182788197 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f608c021070 con 0x55b29461d5f0 2024-04-06T14:17:08.936 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.935+0000 7f60817fa700 1 -- 172.21.15.90:0/4182788197 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f608c005430 con 0x55b29461d5f0 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.935+0000 7f60a4df5b80 1 -- 172.21.15.90:0/4182788197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 msgr2=0x55b29460f1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.935+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 0x55b29460f1b0 secure :-1 s=READY pgs=764 cs=0 l=1 rev1=1 crypto rx=0x7f608c009cb0 tx=0x7f608c01f000 comp rx=0 tx=0).stop 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 -- 172.21.15.90:0/4182788197 shutdown_connections 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29461d5f0 0x55b29460f1b0 unknown :-1 s=CLOSED pgs=764 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294616770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/4182788197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29460b0a0 0x55b294616cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.937 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 -- 172.21.15.90:0/4182788197 >> 172.21.15.90:0/4182788197 conn(0x55b2945696a0 msgr2=0x55b294610820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 -- 172.21.15.90:0/4182788197 shutdown_connections 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 -- 172.21.15.90:0/4182788197 wait complete. 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 -- start start 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.936+0000 7f60a4df5b80 1 Processor -- start 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294682c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294683140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29461abc0 con 0x55b29460b0a0 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29461ad00 con 0x55b29461d5f0 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b29461aec0 con 0x55b29460a780 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f60a4df5b80 10 client.0 _ll_register_callbacks cb 0x55b294501470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.938 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59042/0 (socket says 172.21.15.90:59042) 2024-04-06T14:17:08.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 -- 172.21.15.90:0/3578883674 learned_addr learned my addr 172.21.15.90:0/3578883674 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:08.939 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f609259c700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294682c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 -- 172.21.15.90:0/3578883674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 msgr2=0x55b294682c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294682c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 -- 172.21.15.90:0/3578883674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 msgr2=0x55b294683140 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294683140 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 -- 172.21.15.90:0/3578883674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6094069180 con 0x55b29461d5f0 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6092d9d700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f608c00b770 tx=0x7f608c003c30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4a8bc9e714f5e00 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:08.941 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.937+0000 7f6079ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f608c00e040 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f60840cf090 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f608c023070 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f608c005460 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 10 client.27074 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b2943e1cd0 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f608c049020 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f608c031020 con 0x55b29461d5f0 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 10 client.27074 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 10 client.27074 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 20 client.27074 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:08.942 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b29468a5e0 0x55b29468ca00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 4 client.27074 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55b2943aacf0 con 0x55b29468a5e0 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.938+0000 7f609259c700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b29468a5e0 0x55b29468ca00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f609259c700 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b29468a5e0 0x55b29468ca00 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=669916f47f0171a9 server_cookie=431389e26335ccba in_seq=0 out_seq=0 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f6079ffb700 10 client.27074 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f6079ffb700 1 -- 172.21.15.90:0/3578883674 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55b2943aacf0 con 0x55b29468a5e0 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f6079ffb700 10 client.27074 handle_command_reply: tid=0 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 client.27074 shutdown 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 msgr2=0x55b29461a5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f608c00b770 tx=0x7f608c003c30 comp rx=0 tx=0).stop 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 shutdown_connections 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b29461d5f0 0x55b29461a5f0 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b29468a5e0 0x55b29468ca00 crc :-1 s=READY pgs=202 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.943 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b29460a780 0x55b294682c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 --2- 172.21.15.90:0/3578883674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b29460b0a0 0x55b294683140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 >> 172.21.15.90:0/3578883674 conn(0x55b2945696a0 msgr2=0x55b29460a000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f6079ffb700 0 client.27074 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 shutdown_connections 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 1 -- 172.21.15.90:0/3578883674 wait complete. 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.939+0000 7f60a4df5b80 20 client.27074 trim_cache size 0 max 16384 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:17:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:17:08.945 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:08.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.940+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 msgr2=0x7f607c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.940+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 0x7f607c0a3750 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f608c009fd0 tx=0x7f608c00b040 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.940+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 msgr2=0x7f6094195070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.940+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094195070 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f6084012be0 tx=0x7f6084012ef0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 shutdown_connections 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f607c0a1290 0x7f607c0a3750 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6094105da0 0x7f6094199900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f60940fff00 0x7f6094195070 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 --2- 172.21.15.90:0/3833392641 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60941008c0 0x7f60941955b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.941+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 >> 172.21.15.90:0/3833392641 conn(0x7f60940769b0 msgr2=0x7f60940fece0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.942+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 shutdown_connections 2024-04-06T14:17:08.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:08.942+0000 7f6098c41700 1 -- 172.21.15.90:0/3833392641 wait complete. 2024-04-06T14:17:08.960 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:17:08.960 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:17:08.989 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:08 smithi090 ceph-mon[43420]: pgmap v365: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 84 KiB/s rd, 22 MiB/s wr, 1.73k op/s 2024-04-06T14:17:09.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 -- 172.21.15.90:0/2705057121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d4104600 msgr2=0x7fe8d410c180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 --2- 172.21.15.90:0/2705057121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d4104600 0x7fe8d410c180 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fe8cc00aaa0 tx=0x7fe8cc00adb0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 -- 172.21.15.90:0/2705057121 shutdown_connections 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 --2- 172.21.15.90:0/2705057121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d4104600 0x7fe8d410c180 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 --2- 172.21.15.90:0/2705057121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d40ff8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 --2- 172.21.15.90:0/2705057121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8d40ffe90 0x7fe8d4103ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 -- 172.21.15.90:0/2705057121 >> 172.21.15.90:0/2705057121 conn(0x7fe8d40760a0 msgr2=0x7fe8d40fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 -- 172.21.15.90:0/2705057121 shutdown_connections 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.147+0000 7fe8da212700 1 -- 172.21.15.90:0/2705057121 wait complete. 2024-04-06T14:17:09.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.148+0000 7fe8da212700 1 Processor -- start 2024-04-06T14:17:09.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.149+0000 7fe8da212700 1 -- start start 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d4194c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8d4104600 0x7fe8d4195140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 0x7fe8d41a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8d41a5790 con 0x7fe8d4104600 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8d41a5900 con 0x7fe8d4195680 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8da212700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8d41a5ac0 con 0x7fe8d40ff4e0 2024-04-06T14:17:09.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d4194c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d37fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d4194c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39920/0 (socket says 172.21.15.90:39920) 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d37fe700 1 -- 172.21.15.90:0/3010579063 learned_addr learned my addr 172.21.15.90:0/3010579063 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 0x7fe8d41a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 -- 172.21.15.90:0/3010579063 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 msgr2=0x7fe8d4194c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d4194c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 -- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8d4104600 msgr2=0x7fe8d4195140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8d4104600 0x7fe8d4195140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 -- 172.21.15.90:0/3010579063 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8cc00a700 con 0x7fe8d4195680 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.150+0000 7fe8d3fff700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 0x7fe8d41a51c0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fe8cc00aaa0 tx=0x7fe8cc00cd40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.151+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8cc003bd0 con 0x7fe8d4195680 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.151+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8cc00ed50 con 0x7fe8d4195680 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.151+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8cc005700 con 0x7fe8d4195680 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.151+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8d41a5d40 con 0x7fe8d4195680 2024-04-06T14:17:09.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.151+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8d41a6110 con 0x7fe8d4195680 2024-04-06T14:17:09.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.152+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe8d41078d0 con 0x7fe8d4195680 2024-04-06T14:17:09.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.153+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fe8cc00e890 con 0x7fe8d4195680 2024-04-06T14:17:09.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.154+0000 7fe8d17fa700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 0x7fe8c00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.154+0000 7fe8d37fe700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 0x7fe8c00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.154+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe8cc012070 con 0x7fe8d4195680 2024-04-06T14:17:09.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.154+0000 7fe8d37fe700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 0x7fe8c00a3750 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fe8c4009c10 tx=0x7fe8c400b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.159+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fe8cc097590 con 0x7fe8d4195680 2024-04-06T14:17:09.250 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:08 smithi105 ceph-mon[46969]: pgmap v365: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 84 KiB/s rd, 22 MiB/s wr, 1.73k op/s 2024-04-06T14:17:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:08 smithi134 ceph-mon[47549]: pgmap v365: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 84 KiB/s rd, 22 MiB/s wr, 1.73k op/s 2024-04-06T14:17:09.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.533+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fe8d40ffe90 con 0x7fe8d4195680 2024-04-06T14:17:09.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.533+0000 7fe8d17fa700 1 -- 172.21.15.90:0/3010579063 <== mon.2 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fe8cc095e60 con 0x7fe8d4195680 2024-04-06T14:17:09.535 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:17:09.535 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.536+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 msgr2=0x7fe8c00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.536+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 0x7fe8c00a3750 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fe8c4009c10 tx=0x7fe8c400b040 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.536+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 msgr2=0x7fe8d41a51c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.536+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 0x7fe8d41a51c0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7fe8cc00aaa0 tx=0x7fe8cc00cd40 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 shutdown_connections 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe8c00a1290 0x7fe8c00a3750 secure :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fe8c4009c10 tx=0x7fe8c400b040 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8d4195680 0x7fe8d41a51c0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe8d40ff4e0 0x7fe8d4194c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 --2- 172.21.15.90:0/3010579063 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe8d4104600 0x7fe8d4195140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 >> 172.21.15.90:0/3010579063 conn(0x7fe8d40760a0 msgr2=0x7fe8d41025b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 shutdown_connections 2024-04-06T14:17:09.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.537+0000 7fe8da212700 1 -- 172.21.15.90:0/3010579063 wait complete. 2024-04-06T14:17:09.539 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:17:09.555 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:09.556 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:17:09.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 -- 172.21.15.90:0/1341591313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 msgr2=0x7f5e84107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 --2- 172.21.15.90:0/1341591313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e84107db0 secure :-1 s=READY pgs=765 cs=0 l=1 rev1=1 crypto rx=0x7f5e74009990 tx=0x7f5e74009ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 -- 172.21.15.90:0/1341591313 shutdown_connections 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 --2- 172.21.15.90:0/1341591313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e8411c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 --2- 172.21.15.90:0/1341591313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e84107db0 unknown :-1 s=CLOSED pgs=765 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 --2- 172.21.15.90:0/1341591313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.852+0000 7f5e8be96700 1 -- 172.21.15.90:0/1341591313 >> 172.21.15.90:0/1341591313 conn(0x7f5e8406be00 msgr2=0x7f5e8406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.853+0000 7f5e8be96700 1 -- 172.21.15.90:0/1341591313 shutdown_connections 2024-04-06T14:17:09.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.853+0000 7f5e8be96700 1 -- 172.21.15.90:0/1341591313 wait complete. 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 Processor -- start 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 -- start start 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841a6580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e841aa920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e841aaf60 con 0x7f5e8410f4b0 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e841ab0d0 con 0x7f5e8411a590 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8be96700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5e841ab290 con 0x7f5e8410f920 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e89c32700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e89c32700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47480/0 (socket says 172.21.15.90:47480) 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e89c32700 1 -- 172.21.15.90:0/51937782 learned_addr learned my addr 172.21.15.90:0/51937782 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.856 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e8a433700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e841aa920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.854+0000 7f5e89431700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841a6580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 -- 172.21.15.90:0/51937782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 msgr2=0x7f5e841a6580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841a6580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 -- 172.21.15.90:0/51937782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 msgr2=0x7f5e841aa920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e841aa920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 -- 172.21.15.90:0/51937782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5e740095f0 con 0x7f5e8410f4b0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e8a433700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e841aa920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89431700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841a6580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e89c32700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f5e74000c00 tx=0x7f5e74005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e7affd700 1 -- 172.21.15.90:0/51937782 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e7401d070 con 0x7f5e8410f4b0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e7affd700 1 -- 172.21.15.90:0/51937782 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5e74022470 con 0x7f5e8410f4b0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e7affd700 1 -- 172.21.15.90:0/51937782 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e74017650 con 0x7f5e8410f4b0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5e841ab510 con 0x7f5e8410f4b0 2024-04-06T14:17:09.857 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.855+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5e841ab790 con 0x7f5e8410f4b0 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 Processor -- start 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 -- start start 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e007db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e00988b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009b1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dffd5990 con 0x55f0e0098df0 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dffd3350 con 0x55f0e007d780 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dffcec40 con 0x55f0e007e0a0 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e89c32700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e00988b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e007db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e89c32700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e00988b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39942/0 (socket says 172.21.15.90:39942) 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.861+0000 7f5e89c32700 1 -- 172.21.15.90:0/3270215226 learned_addr learned my addr 172.21.15.90:0/3270215226 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 -- 172.21.15.90:0/3270215226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 msgr2=0x55f0e00988b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e00988b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e89431700 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009b1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 -- 172.21.15.90:0/3270215226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 msgr2=0x55f0e009b1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009b1e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e7affd700 1 -- 172.21.15.90:0/51937782 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5e740225e0 con 0x7f5e8410f4b0 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 -- 172.21.15.90:0/3270215226 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f0dfe503b0 con 0x55f0e007d780 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e89431700 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009b1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e007db60 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f5e80009f90 tx=0x7f5e80011720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b102f750af6c7187 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e6ffff700 1 -- 172.21.15.90:0/3270215226 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80004310 con 0x55f0e007d780 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e6ffff700 1 -- 172.21.15.90:0/3270215226 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5e80004470 con 0x55f0e007d780 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e6ffff700 1 -- 172.21.15.90:0/3270215226 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80003680 con 0x55f0e007d780 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.862+0000 7f5e7affd700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 0x7f5e700a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e7affd700 1 -- 172.21.15.90:0/51937782 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5e740cde30 con 0x7f5e8410f4b0 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e89c32700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 0x7f5e700a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3270215226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 msgr2=0x55f0e007db60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e007db60 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f5e80009f90 tx=0x7f5e80011720 comp rx=0 tx=0).stop 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3270215226 shutdown_connections 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009b1e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e007db60 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3270215226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e00988b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3270215226 >> 172.21.15.90:0/3270215226 conn(0x55f0dffdc6a0 msgr2=0x55f0e0078a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e89c32700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 0x7f5e700a3850 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f5e74022cd0 tx=0x7f5e740a1040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3270215226 shutdown_connections 2024-04-06T14:17:09.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.863+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3270215226 wait complete. 2024-04-06T14:17:09.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 -- start start 2024-04-06T14:17:09.869 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 Processor -- start 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e008c6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e0098df0 0x55f0e00874e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dff6ad80 con 0x55f0e007e0a0 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dff6aa20 con 0x55f0e0098df0 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0dfe526a0 con 0x55f0e007d780 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.864+0000 7f5e9804ab80 10 client.0 _ll_register_callbacks cb 0x55f0dff74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39958/0 (socket says 172.21.15.90:39958) 2024-04-06T14:17:09.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 -- 172.21.15.90:0/3307704144 learned_addr learned my addr 172.21.15.90:0/3307704144 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e0098df0 0x55f0e00874e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 -- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e0098df0 msgr2=0x55f0e00874e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e0098df0 0x55f0e00874e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 -- 172.21.15.90:0/3307704144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 msgr2=0x55f0e008c6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e008c6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 -- 172.21.15.90:0/3307704144 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5e80009c70 con 0x55f0e007d780 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e8a433700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f5e800052e0 tx=0x7f5e800053e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7fa309c4d3226d1d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e6d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80003d40 con 0x55f0e007d780 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.865+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5e80004750 con 0x55f0e007d780 2024-04-06T14:17:09.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55f0dff8f380 con 0x55f0e007d780 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5e80003ea0 con 0x55f0e007d780 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 10 client.26875 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f0dfdfcb60 con 0x55f0e007d780 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55f0dff8f380 con 0x55f0e007d780 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5e80030760 con 0x55f0e007d780 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 10 client.26875 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 10 client.26875 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 20 client.26875 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:09.872 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e0097650 0x55f0e0097a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 4 client.26875 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.866+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55f0dffc7700 con 0x55f0e0097650 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.867+0000 7f5e89431700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e0097650 0x55f0e0097a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.867+0000 7f5e89431700 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e0097650 0x55f0e0097a60 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=6f52415ec53c3ce5 server_cookie=51fe8296f0665f5a in_seq=0 out_seq=0 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.867+0000 7f5e6d7fa700 10 client.26875 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/3307704144 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55f0dffc7700 con 0x55f0e0097650 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e6d7fa700 10 client.26875 handle_command_reply: tid=0 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 client.26875 shutdown 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 msgr2=0x55f0e008c180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.873 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f5e800052e0 tx=0x7f5e800053e0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 shutdown_connections 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e0097650 0x55f0e0097a60 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e0098df0 0x55f0e00874e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007d780 0x55f0e008c180 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/3307704144 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e008c6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 >> 172.21.15.90:0/3307704144 conn(0x55f0dffdc6a0 msgr2=0x55f0e0077c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.874 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.868+0000 7f5e6d7fa700 0 client.26875 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:09.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.871+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 shutdown_connections 2024-04-06T14:17:09.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.871+0000 7f5e9804ab80 1 -- 172.21.15.90:0/3307704144 wait complete. 2024-04-06T14:17:09.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.871+0000 7f5e9804ab80 20 client.26875 trim_cache size 0 max 16384 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 Processor -- start 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 -- start start 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e00783a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 0x55f0e008bda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e0088b20 con 0x55f0e007e0a0 2024-04-06T14:17:09.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e007e4b0 con 0x55f0e007d780 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.888+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e0098c80 con 0x55f0e0098df0 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 0x55f0e008bda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e89431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e89431700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47508/0 (socket says 172.21.15.90:47508) 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e89431700 1 -- 172.21.15.90:0/1104428184 learned_addr learned my addr 172.21.15.90:0/1104428184 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 0x55f0e008bda0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39960/0 (socket says 172.21.15.90:39960) 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e00783a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 -- 172.21.15.90:0/1104428184 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 msgr2=0x55f0e008bda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 0x55f0e008bda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 -- 172.21.15.90:0/1104428184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 msgr2=0x55f0e00788e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e8a433700 1 -- 172.21.15.90:0/1104428184 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f0dff94ea0 con 0x55f0e007d780 2024-04-06T14:17:09.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.889+0000 7f5e89431700 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:09.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.891+0000 7f5e8a433700 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e00783a0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f5e80030b40 tx=0x7f5e80000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ce34345ad1e5174f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.891+0000 7f5e78ff9700 1 -- 172.21.15.90:0/1104428184 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80024dc0 con 0x55f0e007d780 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.891+0000 7f5e78ff9700 1 -- 172.21.15.90:0/1104428184 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5e800308c0 con 0x55f0e007d780 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.891+0000 7f5e78ff9700 1 -- 172.21.15.90:0/1104428184 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80024910 con 0x55f0e007d780 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- 172.21.15.90:0/1104428184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 msgr2=0x55f0e00783a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e00783a0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f5e80030b40 tx=0x7f5e80000bc0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- 172.21.15.90:0/1104428184 shutdown_connections 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e0098df0 0x55f0e008bda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e00783a0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/1104428184 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e007e0a0 0x55f0e00788e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- 172.21.15.90:0/1104428184 >> 172.21.15.90:0/1104428184 conn(0x55f0dffdc6a0 msgr2=0x55f0e0082770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- 172.21.15.90:0/1104428184 shutdown_connections 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- 172.21.15.90:0/1104428184 wait complete. 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 -- start start 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.892+0000 7f5e9804ab80 1 Processor -- start 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e009eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e009a370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e009f790 con 0x55f0e0098df0 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e009adf0 con 0x55f0e007d780 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f0e009afb0 con 0x55f0e007e0a0 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e9804ab80 10 client.0 _ll_register_callbacks cb 0x55f0dff74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47512/0 (socket says 172.21.15.90:47512) 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 -- 172.21.15.90:0/502770874 learned_addr learned my addr 172.21.15.90:0/502770874 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e009eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e89431700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e009a370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 -- 172.21.15.90:0/502770874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 msgr2=0x55f0e009a370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e009a370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 -- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 msgr2=0x55f0e009eeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e009eeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.893+0000 7f5e8a433700 1 -- 172.21.15.90:0/502770874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55f0dfdfcb60 con 0x55f0e0098df0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e8a433700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f5e80030ea0 tx=0x7f5e80005a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1d6c7bd884bbf5ea server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e6d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5e80003e80 con 0x55f0e0098df0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5e80000e40 con 0x55f0e0098df0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5e80004470 con 0x55f0e0098df0 2024-04-06T14:17:09.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5e80030400 con 0x55f0e0098df0 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e9804ab80 10 client.17373 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.894+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f0dfe54cd0 con 0x55f0e0098df0 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5e8004d020 con 0x55f0e0098df0 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5e80048070 con 0x55f0e0098df0 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 10 client.17373 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 10 client.17373 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 20 client.17373 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e00ec160 0x55f0e00ee580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 4 client.17373 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.895+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55f0dfe1dcf0 con 0x55f0e00ec160 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.896+0000 7f5e89431700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e00ec160 0x55f0e00ee580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:09.900 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.896+0000 7f5e89431700 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e00ec160 0x55f0e00ee580 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=a0b341ffae998462 server_cookie=b9cf0a362815a30a in_seq=0 out_seq=0 2024-04-06T14:17:09.901 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:09 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3010579063' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:09.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.899+0000 7f5e6d7fa700 10 client.17373 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:09.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.899+0000 7f5e6d7fa700 1 -- 172.21.15.90:0/502770874 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55f0dfe1dcf0 con 0x55f0e00ec160 2024-04-06T14:17:09.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.899+0000 7f5e6d7fa700 10 client.17373 handle_command_reply: tid=0 2024-04-06T14:17:09.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.901+0000 7f5e9804ab80 1 client.17373 shutdown 2024-04-06T14:17:09.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.901+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 msgr2=0x55f0e009a8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.901+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 secure :-1 s=READY pgs=767 cs=0 l=1 rev1=1 crypto rx=0x7f5e80030ea0 tx=0x7f5e80005a00 comp rx=0 tx=0).stop 2024-04-06T14:17:09.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 shutdown_connections 2024-04-06T14:17:09.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55f0e00ec160 0x55f0e00ee580 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f0e0098df0 0x55f0e009a8b0 unknown :-1 s=CLOSED pgs=767 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f0e007d780 0x55f0e009eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 --2- 172.21.15.90:0/502770874 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f0e007e0a0 0x55f0e009a370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 >> 172.21.15.90:0/502770874 conn(0x55f0dffdc6a0 msgr2=0x55f0e0085d90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e6d7fa700 0 client.17373 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:09.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 shutdown_connections 2024-04-06T14:17:09.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 1 -- 172.21.15.90:0/502770874 wait complete. 2024-04-06T14:17:09.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e9804ab80 20 client.17373 trim_cache size 0 max 16384 2024-04-06T14:17:09.905 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 msgr2=0x7f5e700a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 0x7f5e700a3850 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f5e74022cd0 tx=0x7f5e740a1040 comp rx=0 tx=0).stop 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 msgr2=0x7f5e841a6040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.902+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 secure :-1 s=READY pgs=766 cs=0 l=1 rev1=1 crypto rx=0x7f5e74000c00 tx=0x7f5e74005740 comp rx=0 tx=0).stop 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 shutdown_connections 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5e700a1390 0x7f5e700a3850 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e8411a590 0x7f5e841aa920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5e8410f4b0 0x7f5e841a6040 unknown :-1 s=CLOSED pgs=766 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 --2- 172.21.15.90:0/51937782 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5e8410f920 0x7f5e841a6580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:09.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 >> 172.21.15.90:0/51937782 conn(0x7f5e8406be00 msgr2=0x7f5e84118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:09.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 shutdown_connections 2024-04-06T14:17:09.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:09.903+0000 7f5e8be96700 1 -- 172.21.15.90:0/51937782 wait complete. 2024-04-06T14:17:09.923 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:09.923 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:17:10.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 -- 172.21.15.90:0/935502609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc105da0 msgr2=0x7fdadc106180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 --2- 172.21.15.90:0/935502609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc105da0 0x7fdadc106180 secure :-1 s=READY pgs=768 cs=0 l=1 rev1=1 crypto rx=0x7fdacc0099c0 tx=0x7fdacc009cd0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 -- 172.21.15.90:0/935502609 shutdown_connections 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 --2- 172.21.15.90:0/935502609 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc1008c0 0x7fdadc10cad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 --2- 172.21.15.90:0/935502609 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc105da0 0x7fdadc106180 unknown :-1 s=CLOSED pgs=768 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 --2- 172.21.15.90:0/935502609 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc0fff00 0x7fdadc100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 -- 172.21.15.90:0/935502609 >> 172.21.15.90:0/935502609 conn(0x7fdadc0769b0 msgr2=0x7fdadc076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 -- 172.21.15.90:0/935502609 shutdown_connections 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.112+0000 7fdae1616700 1 -- 172.21.15.90:0/935502609 wait complete. 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 Processor -- start 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 -- start start 2024-04-06T14:17:10.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 0x7fdadc101cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc105da0 0x7fdadc102750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdadc1a55d0 con 0x7fdadc1008c0 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdadc1a5740 con 0x7fdadc105da0 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdae1616700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdadc1a5900 con 0x7fdadc0fff00 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47530/0 (socket says 172.21.15.90:47530) 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 -- 172.21.15.90:0/998300730 learned_addr learned my addr 172.21.15.90:0/998300730 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc105da0 0x7fdadc102750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdadaffd700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 0x7fdadc101cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 -- 172.21.15.90:0/998300730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 msgr2=0x7fdadc101cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 0x7fdadc101cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 -- 172.21.15.90:0/998300730 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc105da0 msgr2=0x7fdadc102750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc105da0 0x7fdadc102750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.113+0000 7fdad3fff700 1 -- 172.21.15.90:0/998300730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdacc0095f0 con 0x7fdadc1008c0 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdadaffd700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 0x7fdadc101cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdad3fff700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7fdac400d950 tx=0x7fdac400dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdad8ff9700 1 -- 172.21.15.90:0/998300730 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdac4009940 con 0x7fdadc1008c0 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdad8ff9700 1 -- 172.21.15.90:0/998300730 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdac4010460 con 0x7fdadc1008c0 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdadc1a5be0 con 0x7fdadc1008c0 2024-04-06T14:17:10.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.114+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdadc1a60f0 con 0x7fdadc1008c0 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.118+0000 7fdaed7cab80 1 Processor -- start 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.118+0000 7fdaed7cab80 1 -- start start 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be399370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3957c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3ac060 0x55a8be3ae450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be2e8990 con 0x55a8be3ac060 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be2e6350 con 0x55a8be398f90 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be2e1c40 con 0x55a8be3998b0 2024-04-06T14:17:10.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadaffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3957c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadaffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3957c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39986/0 (socket says 172.21.15.90:39986) 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be399370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadaffd700 1 -- 172.21.15.90:0/2577078994 learned_addr learned my addr 172.21.15.90:0/2577078994 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 -- 172.21.15.90:0/2577078994 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 msgr2=0x55a8be3957c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3957c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 -- 172.21.15.90:0/2577078994 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3ac060 msgr2=0x55a8be3ae450 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3ac060 0x55a8be3ae450 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.119+0000 7fdadb7fe700 1 -- 172.21.15.90:0/2577078994 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a8be1633b0 con 0x55a8be398f90 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.120+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be399370 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fdadc102c90 tx=0x7fdad4009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a7c1ba2e20481490 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.120+0000 7fdad17fa700 1 -- 172.21.15.90:0/2577078994 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdad400e040 con 0x55a8be398f90 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.120+0000 7fdad17fa700 1 -- 172.21.15.90:0/2577078994 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdad400f040 con 0x55a8be398f90 2024-04-06T14:17:10.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.120+0000 7fdad17fa700 1 -- 172.21.15.90:0/2577078994 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdad4004360 con 0x55a8be398f90 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdad8ff9700 1 -- 172.21.15.90:0/998300730 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdac4009ca0 con 0x7fdadc1008c0 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2577078994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 msgr2=0x55a8be399370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be399370 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7fdadc102c90 tx=0x7fdad4009920 comp rx=0 tx=0).stop 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2577078994 shutdown_connections 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3ac060 0x55a8be3ae450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be399370 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2577078994 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3957c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.121+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2577078994 >> 172.21.15.90:0/2577078994 conn(0x55a8be2ef6a0 msgr2=0x55a8be38ba40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdad8ff9700 1 -- 172.21.15.90:0/998300730 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fdac40108d0 con 0x7fdadc1008c0 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2577078994 shutdown_connections 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2577078994 wait complete. 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdaed7cab80 1 -- start start 2024-04-06T14:17:10.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdaed7cab80 1 Processor -- start 2024-04-06T14:17:10.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.122+0000 7fdad8ff9700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 0x7fdac80a3660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad8ff9700 1 -- 172.21.15.90:0/998300730 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fdac40ccf70 con 0x7fdadc1008c0 2024-04-06T14:17:10.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be39b430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3a42b0 0x55a8be3a6740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be27dd80 con 0x55a8be3a42b0 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be27da20 con 0x55a8be398f90 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be1656a0 con 0x55a8be3998b0 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdaed7cab80 10 client.0 _ll_register_callbacks cb 0x55a8be287470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39990/0 (socket says 172.21.15.90:39990) 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 -- 172.21.15.90:0/2779240806 learned_addr learned my addr 172.21.15.90:0/2779240806 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3a42b0 0x55a8be3a6740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 -- 172.21.15.90:0/2779240806 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 msgr2=0x55a8be39b430 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be39b430 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 -- 172.21.15.90:0/2779240806 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3a42b0 msgr2=0x55a8be3a6740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3a42b0 0x55a8be3a6740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.123+0000 7fdad3fff700 1 -- 172.21.15.90:0/2779240806 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdad4011230 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 0x7fdac80a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdad3fff700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fdac40ca700 tx=0x7fdac4099cd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1bac1a94115a0a77 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdac27fc700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdac409de90 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fdac40a24e0 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.124+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a8be2a2380 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdac4096770 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 10 client.26890 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a8be10fb60 con 0x55a8be3998b0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 0x7fdac80a3660 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fdad400b010 tx=0x7fdad400b040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55a8be2a2380 con 0x55a8be3998b0 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fdac4099550 con 0x55a8be3998b0 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 10 client.26890 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 10 client.26890 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 20 client.26890 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3aab20 0x55a8be3aaf30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 4 client.26890 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.125+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a8be2da700 con 0x55a8be3aab20 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.126+0000 7fdadaffd700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3aab20 0x55a8be3aaf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.126+0000 7fdadaffd700 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3aab20 0x55a8be3aaf30 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=4c0c1440a9dc30bf server_cookie=7dc3b4d2801bc598 in_seq=0 out_seq=0 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.126+0000 7fdac27fc700 10 client.26890 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdac27fc700 1 -- 172.21.15.90:0/2779240806 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a8be2da700 con 0x55a8be3aab20 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdac27fc700 10 client.26890 handle_command_reply: tid=0 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 client.26890 shutdown 2024-04-06T14:17:10.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 msgr2=0x55a8be39b970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7fdac40ca700 tx=0x7fdac4099cd0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 shutdown_connections 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3aab20 0x55a8be3aaf30 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3a42b0 0x55a8be3a6740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be39b430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/2779240806 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be39b970 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 >> 172.21.15.90:0/2779240806 conn(0x55a8be2ef6a0 msgr2=0x55a8be38ac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdac27fc700 0 client.26890 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 shutdown_connections 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 1 -- 172.21.15.90:0/2779240806 wait complete. 2024-04-06T14:17:10.132 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.127+0000 7fdaed7cab80 20 client.26890 trim_cache size 0 max 16384 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 Processor -- start 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 -- start start 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be3b2480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 0x55a8be38f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 0x55a8be3939d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be3956c0 con 0x55a8be3998b0 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be3a8370 con 0x55a8be398f90 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be39a2a0 con 0x55a8be3a42b0 2024-04-06T14:17:10.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 0x55a8be38f7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdad3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 0x55a8be38f7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47554/0 (socket says 172.21.15.90:47554) 2024-04-06T14:17:10.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdad3fff700 1 -- 172.21.15.90:0/207746123 learned_addr learned my addr 172.21.15.90:0/207746123 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 0x55a8be3939d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.142+0000 7fdadaffd700 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be3b2480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 -- 172.21.15.90:0/207746123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 msgr2=0x55a8be3b2480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be3b2480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 -- 172.21.15.90:0/207746123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 msgr2=0x55a8be38f7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 0x55a8be38f7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 -- 172.21.15.90:0/207746123 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a8be2a7ea0 con 0x55a8be3a42b0 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 0x55a8be3939d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7fdad4009370 tx=0x7fdad4009510 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c1cf6795a7f8c901 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdad1ffb700 1 -- 172.21.15.90:0/207746123 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdad400e040 con 0x55a8be3a42b0 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdad1ffb700 1 -- 172.21.15.90:0/207746123 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdad4021070 con 0x55a8be3a42b0 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.143+0000 7fdad1ffb700 1 -- 172.21.15.90:0/207746123 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdad4007b40 con 0x55a8be3a42b0 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 -- 172.21.15.90:0/207746123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 msgr2=0x55a8be3939d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 0x55a8be3939d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7fdad4009370 tx=0x7fdad4009510 comp rx=0 tx=0).stop 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 -- 172.21.15.90:0/207746123 shutdown_connections 2024-04-06T14:17:10.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3a42b0 0x55a8be3939d0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be398f90 0x55a8be3b2480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/207746123 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be3998b0 0x55a8be38f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 -- 172.21.15.90:0/207746123 >> 172.21.15.90:0/207746123 conn(0x55a8be2ef6a0 msgr2=0x55a8be3ab870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 -- 172.21.15.90:0/207746123 shutdown_connections 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.144+0000 7fdaed7cab80 1 -- 172.21.15.90:0/207746123 wait complete. 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 -- start start 2024-04-06T14:17:10.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 Processor -- start 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3ae040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be3a42b0 0x55a8be3ae580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be3fe510 con 0x55a8be398f90 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be3fe650 con 0x55a8be3a42b0 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a8be3fe7c0 con 0x55a8be3998b0 2024-04-06T14:17:10.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdaed7cab80 10 client.0 _ll_register_callbacks cb 0x55a8be287470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47568/0 (socket says 172.21.15.90:47568) 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 -- 172.21.15.90:0/3062649688 learned_addr learned my addr 172.21.15.90:0/3062649688 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdad3fff700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3ae040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadb7fe700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be3a42b0 0x55a8be3ae580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 -- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 msgr2=0x55a8be3ae040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3ae040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 -- 172.21.15.90:0/3062649688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be3a42b0 msgr2=0x55a8be3ae580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.145+0000 7fdadaffd700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be3a42b0 0x55a8be3ae580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdadaffd700 1 -- 172.21.15.90:0/3062649688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55a8be1633b0 con 0x55a8be398f90 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdadaffd700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7fdacc00b580 tx=0x7fdacc005040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4acf21a19e16b084 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdac27fc700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdacc041040 con 0x55a8be398f90 2024-04-06T14:17:10.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fdacc047c70 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fdad400f040 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdacc047e30 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdaed7cab80 10 client.17388 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.146+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a8be167cd0 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdad400f040 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fdacc00bd60 con 0x55a8be398f90 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 10 client.17388 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 10 client.17388 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 20 client.17388 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3fff70 0x55a8be402360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 4 client.17388 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55a8be130cf0 con 0x55a8be3fff70 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.147+0000 7fdad3fff700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3fff70 0x55a8be402360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdac27fc700 10 client.17388 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdad3fff700 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3fff70 0x55a8be402360 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f32784c602617ec9 server_cookie=1bada3cd5b03dff2 in_seq=0 out_seq=0 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdac27fc700 1 -- 172.21.15.90:0/3062649688 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55a8be130cf0 con 0x55a8be3fff70 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdac27fc700 10 client.17388 handle_command_reply: tid=0 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 client.17388 shutdown 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 msgr2=0x55a8be3adb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 secure :-1 s=READY pgs=770 cs=0 l=1 rev1=1 crypto rx=0x7fdacc00b580 tx=0x7fdacc005040 comp rx=0 tx=0).stop 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 shutdown_connections 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55a8be3fff70 0x55a8be402360 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a8be3a42b0 0x55a8be3ae580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a8be398f90 0x55a8be3adb00 unknown :-1 s=CLOSED pgs=770 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 --2- 172.21.15.90:0/3062649688 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a8be3998b0 0x55a8be3ae040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 >> 172.21.15.90:0/3062649688 conn(0x55a8be2ef6a0 msgr2=0x55a8be393910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdac27fc700 0 client.17388 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 shutdown_connections 2024-04-06T14:17:10.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 1 -- 172.21.15.90:0/3062649688 wait complete. 2024-04-06T14:17:10.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.148+0000 7fdaed7cab80 20 client.17388 trim_cache size 0 max 16384 2024-04-06T14:17:10.153 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:10.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 msgr2=0x7fdac80a3660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 0x7fdac80a3660 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fdad400b010 tx=0x7fdad400b040 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 msgr2=0x7fdadc1021f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 secure :-1 s=READY pgs=769 cs=0 l=1 rev1=1 crypto rx=0x7fdac400d950 tx=0x7fdac400dc60 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 shutdown_connections 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fdac80a11a0 0x7fdac80a3660 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdadc105da0 0x7fdadc102750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fdadc0fff00 0x7fdadc101cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 --2- 172.21.15.90:0/998300730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fdadc1008c0 0x7fdadc1021f0 unknown :-1 s=CLOSED pgs=769 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 >> 172.21.15.90:0/998300730 conn(0x7fdadc0769b0 msgr2=0x7fdadc103f90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 shutdown_connections 2024-04-06T14:17:10.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.149+0000 7fdae1616700 1 -- 172.21.15.90:0/998300730 wait complete. 2024-04-06T14:17:10.166 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:10.167 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:17:10.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:09 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3010579063' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:10.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.344+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1104683728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c54100820 msgr2=0x7f1c54104860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.344 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.344+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1104683728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c54100820 0x7f1c54104860 secure :-1 s=READY pgs=771 cs=0 l=1 rev1=1 crypto rx=0x7f1c48009a10 tx=0x7f1c48009d20 comp rx=0 tx=0).stop 2024-04-06T14:17:10.347 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1104683728 shutdown_connections 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1104683728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c54104f90 0x7f1c5410cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1104683728 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c540ffe70 0x7f1c54100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1104683728 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c54100820 0x7f1c54104860 unknown :-1 s=CLOSED pgs=771 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1104683728 >> 172.21.15.90:0/1104683728 conn(0x7f1c54076e10 msgr2=0x7f1c54077220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1104683728 shutdown_connections 2024-04-06T14:17:10.348 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.347+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1104683728 wait complete. 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 Processor -- start 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 -- start start 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c540ffe70 0x7f1c54072450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c5406c540 0x7f1c5406c9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c5406d1c0 con 0x7f1c540ffe70 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c5406d330 con 0x7f1c5406c540 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.348+0000 7f1c59d3c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c541a4170 con 0x7f1c54104f90 2024-04-06T14:17:10.351 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40028/0 (socket says 172.21.15.90:40028) 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1520074406 learned_addr learned my addr 172.21.15.90:0/1520074406 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c53fff700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c5406c540 0x7f1c5406c9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1520074406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c5406c540 msgr2=0x7f1c5406c9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c5406c540 0x7f1c5406c9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c540ffe70 msgr2=0x7f1c54072450 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c540ffe70 0x7f1c54072450 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1520074406 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c480095f0 con 0x7f1c54104f90 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.349+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1c48006010 tx=0x7f1c48005e10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.350+0000 7f1c50ff9700 1 -- 172.21.15.90:0/1520074406 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4800bb00 con 0x7f1c54104f90 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.350+0000 7f1c50ff9700 1 -- 172.21.15.90:0/1520074406 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1c480041b0 con 0x7f1c54104f90 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.350+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c541a43f0 con 0x7f1c54104f90 2024-04-06T14:17:10.352 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.350+0000 7f1c50ff9700 1 -- 172.21.15.90:0/1520074406 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c48016430 con 0x7f1c54104f90 2024-04-06T14:17:10.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.350+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c541a4950 con 0x7f1c54104f90 2024-04-06T14:17:10.353 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.353+0000 7f1c50ff9700 1 -- 172.21.15.90:0/1520074406 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1c48041450 con 0x7f1c54104f90 2024-04-06T14:17:10.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.353+0000 7f1c50ff9700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 0x7f1c3c0a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.354+0000 7f1c50ff9700 1 -- 172.21.15.90:0/1520074406 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1c480f0ff0 con 0x7f1c54104f90 2024-04-06T14:17:10.354 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.354+0000 7f1c537fe700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 0x7f1c3c0a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.354+0000 7f1c537fe700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 0x7f1c3c0a3810 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f1c4400ba10 tx=0x7f1c4400b3f0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 Processor -- start 2024-04-06T14:17:10.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 -- start start 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d0b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9da8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dd1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec928990 con 0x55b5ec9dade0 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec926350 con 0x55b5ec9d10a0 2024-04-06T14:17:10.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.355+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec921c40 con 0x55b5ec9d0780 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dd1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dd1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47578/0 (socket says 172.21.15.90:47578) 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c53fff700 1 -- 172.21.15.90:0/1466636312 learned_addr learned my addr 172.21.15.90:0/1466636312 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c537fe700 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d0b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9da8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1466636312 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d0780 msgr2=0x55b5ec9d0b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d0b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1466636312 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 msgr2=0x55b5ec9dd1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dd1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 -- 172.21.15.90:0/1466636312 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b5ec7a33b0 con 0x55b5ec9d10a0 2024-04-06T14:17:10.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.356+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9da8a0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1c480ee7d0 tx=0x7f1c48009f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7598e8573d1cc48 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.357+0000 7f1c39ffb700 1 -- 172.21.15.90:0/1466636312 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c480be220 con 0x55b5ec9d10a0 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.357+0000 7f1c39ffb700 1 -- 172.21.15.90:0/1466636312 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1c480c2050 con 0x55b5ec9d10a0 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.357+0000 7f1c39ffb700 1 -- 172.21.15.90:0/1466636312 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c480bab10 con 0x55b5ec9d10a0 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/1466636312 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 msgr2=0x55b5ec9da8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9da8a0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1c480ee7d0 tx=0x7f1c48009f60 comp rx=0 tx=0).stop 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/1466636312 shutdown_connections 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dd1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d0b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/1466636312 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9da8a0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/1466636312 >> 172.21.15.90:0/1466636312 conn(0x55b5ec92f6a0 msgr2=0x55b5ec9cba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/1466636312 shutdown_connections 2024-04-06T14:17:10.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.358+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/1466636312 wait complete. 2024-04-06T14:17:10.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 -- start start 2024-04-06T14:17:10.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 Processor -- start 2024-04-06T14:17:10.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9da890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9d8310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec8bdd80 con 0x55b5ec9dade0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec8bda20 con 0x55b5ec9d0780 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec7a56a0 con 0x55b5ec9d10a0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.359+0000 7f1c65ef0b80 10 client.0 _ll_register_callbacks cb 0x55b5ec8c7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40046/0 (socket says 172.21.15.90:40046) 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 -- 172.21.15.90:0/2681833605 learned_addr learned my addr 172.21.15.90:0/2681833605 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c537fe700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9da890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c53fff700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9d8310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 -- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 msgr2=0x55b5ec9da890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9da890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 -- 172.21.15.90:0/2681833605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 msgr2=0x55b5ec9d8310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9d8310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 -- 172.21.15.90:0/2681833605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c541a4950 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c53fff700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9d8310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f1c480a9050 tx=0x7f1c48009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=77b9717e80b04d1e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c32ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c480be680 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1c480be220 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b5ec8e2380 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1c480bd710 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c65ef0b80 10 client.26908 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.360+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b5ec74fb60 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1c480a4030 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1c480dc2c0 con 0x55b5ec9d10a0 2024-04-06T14:17:10.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 10 client.26908 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 10 client.26908 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 20 client.26908 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5ec9ea360 0x55b5ec9ec780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 4 client.26908 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.361+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b5ec91a700 con 0x55b5ec9ea360 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c537fe700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5ec9ea360 0x55b5ec9ec780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c32ffd700 10 client.26908 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:10.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c537fe700 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5ec9ea360 0x55b5ec9ec780 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a6845debde556091 server_cookie=80dffa2e5dc62b08 in_seq=0 out_seq=0 2024-04-06T14:17:10.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c32ffd700 1 -- 172.21.15.90:0/2681833605 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b5ec91a700 con 0x55b5ec9ea360 2024-04-06T14:17:10.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c32ffd700 10 client.26908 handle_command_reply: tid=0 2024-04-06T14:17:10.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.362+0000 7f1c65ef0b80 1 client.26908 shutdown 2024-04-06T14:17:10.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 msgr2=0x55b5ec9d7bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f1c480a9050 tx=0x7f1c48009310 comp rx=0 tx=0).stop 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 shutdown_connections 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9dade0 0x55b5ec9d8310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9da890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5ec9ea360 0x55b5ec9ec780 crc :-1 s=READY pgs=203 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/2681833605 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d7bd0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 >> 172.21.15.90:0/2681833605 conn(0x55b5ec92f6a0 msgr2=0x55b5ec9cac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c32ffd700 0 client.26908 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 shutdown_connections 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/2681833605 wait complete. 2024-04-06T14:17:10.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.363+0000 7f1c65ef0b80 20 client.26908 trim_cache size 0 max 16384 2024-04-06T14:17:10.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 Processor -- start 2024-04-06T14:17:10.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 -- start start 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d7b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d74b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9d79f0 con 0x55b5ec9d0780 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9d8b00 con 0x55b5ec9dade0 2024-04-06T14:17:10.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.377+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9cc7c0 con 0x55b5ec9d10a0 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59194/0 (socket says 172.21.15.90:59194) 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 -- 172.21.15.90:0/4014579339 learned_addr learned my addr 172.21.15.90:0/4014579339 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 -- 172.21.15.90:0/4014579339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 msgr2=0x55b5ec9d74b0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d74b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 -- 172.21.15.90:0/4014579339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d0780 msgr2=0x55b5ec9d7b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d7b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 -- 172.21.15.90:0/4014579339 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b5ec8e7ea0 con 0x55b5ec9dade0 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.378+0000 7f1c53fff700 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c008550 tx=0x7f1c4c007820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=41bd65246b81f26f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.379+0000 7f1c3a7fc700 1 -- 172.21.15.90:0/4014579339 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4c00f040 con 0x55b5ec9dade0 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.379+0000 7f1c3a7fc700 1 -- 172.21.15.90:0/4014579339 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1c4c010040 con 0x55b5ec9dade0 2024-04-06T14:17:10.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.379+0000 7f1c3a7fc700 1 -- 172.21.15.90:0/4014579339 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4c016c50 con 0x55b5ec9dade0 2024-04-06T14:17:10.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.379+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/4014579339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 msgr2=0x55b5ec9ec370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.379+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c008550 tx=0x7f1c4c007820 comp rx=0 tx=0).stop 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/4014579339 shutdown_connections 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9dade0 0x55b5ec9ec370 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d0780 0x55b5ec9d7b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/4014579339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9d74b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/4014579339 >> 172.21.15.90:0/4014579339 conn(0x55b5ec92f6a0 msgr2=0x55b5ec9f10e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/4014579339 shutdown_connections 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/4014579339 wait complete. 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- start start 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 Processor -- start 2024-04-06T14:17:10.385 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9e13f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9e0240 con 0x55b5ec9d10a0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9e03b0 con 0x55b5ec9d0780 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.380+0000 7f1c65ef0b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b5ec9e0570 con 0x55b5ec9dade0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c65ef0b80 10 client.0 _ll_register_callbacks cb 0x55b5ec8c7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c52ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47596/0 (socket says 172.21.15.90:47596) 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40072/0 (socket says 172.21.15.90:40072) 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c52ffd700 1 -- 172.21.15.90:0/3907333226 learned_addr learned my addr 172.21.15.90:0/3907333226 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c537fe700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9e13f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.386 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 -- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 msgr2=0x55b5ec9e13f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9e13f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 -- 172.21.15.90:0/3907333226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 msgr2=0x55b5ec9e1930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 -- 172.21.15.90:0/3907333226 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c4c007e70 con 0x55b5ec9dade0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c52ffd700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c53fff700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c008550 tx=0x7f1c4c007df0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b9625508e7f2faf3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c32ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.381+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c4c00f030 con 0x55b5ec9dade0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1c480c2050 con 0x55b5ec9dade0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(94..94 src has 1..94) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1c4c016ce0 con 0x55b5ec9dade0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1c4c016ea0 con 0x55b5ec9dade0 2024-04-06T14:17:10.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 10 client.26911 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b5ec7a7cd0 con 0x55b5ec9dade0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1c480c2050 con 0x55b5ec9dade0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1c4c0208c0 con 0x55b5ec9dade0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 10 client.26911 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 10 client.26911 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 20 client.26911 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5eca3f3b0 0x55b5eca417d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 4 client.26911 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b5ec770cf0 con 0x55b5eca3f3b0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.382+0000 7f1c537fe700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5eca3f3b0 0x55b5eca417d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c32ffd700 10 client.26911 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c537fe700 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5eca3f3b0 0x55b5eca417d0 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=997a3575e6b13f15 server_cookie=84b7bfd6644a4989 in_seq=0 out_seq=0 2024-04-06T14:17:10.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c32ffd700 1 -- 172.21.15.90:0/3907333226 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b5ec770cf0 con 0x55b5eca3f3b0 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c32ffd700 10 client.26911 handle_command_reply: tid=0 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 client.26911 shutdown 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 msgr2=0x55b5ec9dfb30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f1c4c008550 tx=0x7f1c4c007df0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 shutdown_connections 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b5eca3f3b0 0x55b5eca417d0 crc :-1 s=READY pgs=204 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b5ec9dade0 0x55b5ec9dfb30 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b5ec9d0780 0x55b5ec9e13f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 --2- 172.21.15.90:0/3907333226 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b5ec9d10a0 0x55b5ec9e1930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c32ffd700 0 client.26911 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 >> 172.21.15.90:0/3907333226 conn(0x55b5ec92f6a0 msgr2=0x55b5ec9d2ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 shutdown_connections 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.383+0000 7f1c65ef0b80 1 -- 172.21.15.90:0/3907333226 wait complete. 2024-04-06T14:17:10.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c65ef0b80 20 client.26911 trim_cache size 0 max 16384 2024-04-06T14:17:10.390 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 msgr2=0x7f1c3c0a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 0x7f1c3c0a3810 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f1c4400ba10 tx=0x7f1c4400b3f0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 msgr2=0x7f1c54072990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1c48006010 tx=0x7f1c48005e10 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 shutdown_connections 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1c3c0a1350 0x7f1c3c0a3810 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c5406c540 0x7f1c5406c9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1c540ffe70 0x7f1c54072450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 --2- 172.21.15.90:0/1520074406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1c54104f90 0x7f1c54072990 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:10.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 >> 172.21.15.90:0/1520074406 conn(0x7f1c54076e10 msgr2=0x7f1c540fdd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:10.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 shutdown_connections 2024-04-06T14:17:10.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:10.384+0000 7f1c59d3c700 1 -- 172.21.15.90:0/1520074406 wait complete. 2024-04-06T14:17:10.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:09 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3010579063' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:11.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:10 smithi105 ceph-mon[46969]: pgmap v366: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:11.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:10 smithi090 ceph-mon[43420]: pgmap v366: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:11.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:10 smithi134 ceph-mon[47549]: pgmap v366: 97 pgs: 97 active+clean; 6.8 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:11.403 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:17:11.403 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:17:11.403 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:17:11.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.668+0000 7f1880ab5700 1 -- 172.21.15.90:0/1511611069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c0ffcc0 msgr2=0x7f187c103960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.668+0000 7f1880ab5700 1 --2- 172.21.15.90:0/1511611069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c0ffcc0 0x7f187c103960 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f187400b0a0 tx=0x7f187400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 -- 172.21.15.90:0/1511611069 shutdown_connections 2024-04-06T14:17:11.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 --2- 172.21.15.90:0/1511611069 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c0ffcc0 0x7f187c103960 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 --2- 172.21.15.90:0/1511611069 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f187c105e60 0x7f187c106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 --2- 172.21.15.90:0/1511611069 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ff220 0x7f187c0ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 -- 172.21.15.90:0/1511611069 >> 172.21.15.90:0/1511611069 conn(0x7f187c0761b0 msgr2=0x7f187c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 -- 172.21.15.90:0/1511611069 shutdown_connections 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.669+0000 7f1880ab5700 1 -- 172.21.15.90:0/1511611069 wait complete. 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.670+0000 7f1880ab5700 1 Processor -- start 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.670+0000 7f1880ab5700 1 -- start start 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f187c0ff220 0x7f187c1a3950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 0x7f187c1a3e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 0x7f187c19da80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f187c19e0f0 con 0x7f187c0ff220 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f187c19e260 con 0x7f187c0ffcc0 2024-04-06T14:17:11.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1880ab5700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f187c19e420 con 0x7f187c105e60 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1879d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 0x7f187c1a3e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1879d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 0x7f187c1a3e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59226/0 (socket says 172.21.15.90:59226) 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.671+0000 7f1879d9b700 1 -- 172.21.15.90:0/626403093 learned_addr learned my addr 172.21.15.90:0/626403093 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.672+0000 7f187ad9d700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 0x7f187c19da80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 -- 172.21.15.90:0/626403093 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 msgr2=0x7f187c1a3e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 0x7f187c1a3e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 -- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f187c0ff220 msgr2=0x7f187c1a3950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f187c0ff220 0x7f187c1a3950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 -- 172.21.15.90:0/626403093 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1874009d00 con 0x7f187c105e60 2024-04-06T14:17:11.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.673+0000 7f187ad9d700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 0x7f187c19da80 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f187400b0a0 tx=0x7f18740077f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.675+0000 7f186b7fe700 1 -- 172.21.15.90:0/626403093 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f187400e040 con 0x7f187c105e60 2024-04-06T14:17:11.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.675+0000 7f186b7fe700 1 -- 172.21.15.90:0/626403093 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1874004e60 con 0x7f187c105e60 2024-04-06T14:17:11.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.675+0000 7f186b7fe700 1 -- 172.21.15.90:0/626403093 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1874005330 con 0x7f187c105e60 2024-04-06T14:17:11.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.679+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f187c19e6a0 con 0x7f187c105e60 2024-04-06T14:17:11.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.679+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f187c06a9b0 con 0x7f187c105e60 2024-04-06T14:17:11.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.683+0000 7f188cc69b80 1 Processor -- start 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 -- start start 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5601be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb560b920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560e250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb5559a10 con 0x55dbb5601800 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb55573d0 con 0x55dbb560be60 2024-04-06T14:17:11.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.689+0000 7f188cc69b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb5552cc0 con 0x55dbb5602120 2024-04-06T14:17:11.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.690+0000 7f1879d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5601be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.690+0000 7f1879d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5601be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47628/0 (socket says 172.21.15.90:47628) 2024-04-06T14:17:11.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.690+0000 7f1879d9b700 1 -- 172.21.15.90:0/3782049229 learned_addr learned my addr 172.21.15.90:0/3782049229 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:11.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.690+0000 7f1879d9b700 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560e250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.691 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.690+0000 7f187a59c700 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb560b920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 -- 172.21.15.90:0/3782049229 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 msgr2=0x55dbb560e250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560e250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 -- 172.21.15.90:0/3782049229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 msgr2=0x55dbb5601be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5601be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 -- 172.21.15.90:0/3782049229 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dbb53d4430 con 0x55dbb5602120 2024-04-06T14:17:11.692 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.691+0000 7f187a59c700 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb560b920 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f18700097f0 tx=0x7f1870009b00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a17e7f5e69c09e03 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.693+0000 7f1868ff9700 1 -- 172.21.15.90:0/3782049229 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1870011830 con 0x55dbb5602120 2024-04-06T14:17:11.697 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.693+0000 7f1868ff9700 1 -- 172.21.15.90:0/3782049229 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1870011e50 con 0x55dbb5602120 2024-04-06T14:17:11.698 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.698+0000 7f1868ff9700 1 -- 172.21.15.90:0/3782049229 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1870010380 con 0x55dbb5602120 2024-04-06T14:17:11.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.699+0000 7f186b7fe700 1 -- 172.21.15.90:0/626403093 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f187401b9a0 con 0x7f187c105e60 2024-04-06T14:17:11.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.699+0000 7f186b7fe700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 0x7f18640a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.700 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.700+0000 7f186b7fe700 1 -- 172.21.15.90:0/626403093 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f18740cdf50 con 0x7f187c105e60 2024-04-06T14:17:11.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.703+0000 7f188cc69b80 1 -- 172.21.15.90:0/3782049229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 msgr2=0x55dbb560b920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.706 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.703+0000 7f188cc69b80 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb560b920 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f18700097f0 tx=0x7f1870009b00 comp rx=0 tx=0).stop 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.712+0000 7f1879d9b700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 0x7f18640a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.712+0000 7f1879d9b700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 0x7f18640a3910 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x55dbb5603270 tx=0x7f186c008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.713+0000 7f188cc69b80 1 -- 172.21.15.90:0/3782049229 shutdown_connections 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.713+0000 7f188cc69b80 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560e250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.713+0000 7f188cc69b80 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5601be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.713+0000 7f188cc69b80 1 --2- 172.21.15.90:0/3782049229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb560b920 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.714 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.713+0000 7f188cc69b80 1 -- 172.21.15.90:0/3782049229 >> 172.21.15.90:0/3782049229 conn(0x55dbb5560720 msgr2=0x55dbb55fcb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.723 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.723+0000 7f188cc69b80 1 -- 172.21.15.90:0/3782049229 shutdown_connections 2024-04-06T14:17:11.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.723+0000 7f188cc69b80 1 -- 172.21.15.90:0/3782049229 wait complete. 2024-04-06T14:17:11.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.724+0000 7f188cc69b80 1 -- start start 2024-04-06T14:17:11.724 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.724+0000 7f188cc69b80 1 Processor -- start 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.724+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.724+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5608c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.724+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb5609380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.725+0000 7f188cc69b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb54eee00 con 0x55dbb5601800 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.725+0000 7f188cc69b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb54eeaa0 con 0x55dbb560be60 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.725+0000 7f188cc69b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb53d6720 con 0x55dbb5602120 2024-04-06T14:17:11.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.725+0000 7f188cc69b80 10 client.0 _ll_register_callbacks cb 0x55dbb54f84f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:47640/0 (socket says 172.21.15.90:47640) 2024-04-06T14:17:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 -- 172.21.15.90:0/2033204773 learned_addr learned my addr 172.21.15.90:0/2033204773 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:11.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187ad9d700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb5609380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 -- 172.21.15.90:0/2033204773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 msgr2=0x55dbb5608c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5608c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 -- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 msgr2=0x55dbb5609380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb5609380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 -- 172.21.15.90:0/2033204773 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1870009420 con 0x55dbb5601800 2024-04-06T14:17:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f187a59c700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f187000bbd0 tx=0x7f187000bf10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=faf54a5c0ff86626 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.726+0000 7f1861ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:11.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.731+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1870011630 con 0x55dbb5601800 2024-04-06T14:17:11.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.731+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f1870011790 con 0x55dbb5601800 2024-04-06T14:17:11.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.731+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55dbb5513400 con 0x55dbb5601800 2024-04-06T14:17:11.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.733+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1870011950 con 0x55dbb5601800 2024-04-06T14:17:11.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.737+0000 7f188cc69b80 10 client.17406 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:11.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.737+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dbb5380be0 con 0x55dbb5601800 2024-04-06T14:17:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55dbb5513400 con 0x55dbb5601800 2024-04-06T14:17:11.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f187002d420 con 0x55dbb5601800 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 10 client.17406 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 10 client.17406 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 20 client.17406 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb562ca00 0x55dbb562ee20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 4 client.17406 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:11.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.738+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55dbb554b780 con 0x55dbb562ca00 2024-04-06T14:17:11.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.744+0000 7f1879d9b700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb562ca00 0x55dbb562ee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.746+0000 7f1879d9b700 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb562ca00 0x55dbb562ee20 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4a09b9166087568c server_cookie=8d6a8328e1ea4174 in_seq=0 out_seq=0 2024-04-06T14:17:11.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.747+0000 7f1861ffb700 10 client.17406 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:11.747 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.747+0000 7f1861ffb700 1 -- 172.21.15.90:0/2033204773 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55dbb554b780 con 0x55dbb562ca00 2024-04-06T14:17:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.747+0000 7f1861ffb700 10 client.17406 handle_command_reply: tid=0 2024-04-06T14:17:11.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.747+0000 7f188cc69b80 1 client.17406 shutdown 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 msgr2=0x55dbb560b8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 secure :-1 s=READY pgs=772 cs=0 l=1 rev1=1 crypto rx=0x7f187000bbd0 tx=0x7f187000bf10 comp rx=0 tx=0).stop 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 shutdown_connections 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb5609380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb560b8b0 unknown :-1 s=CLOSED pgs=772 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb562ca00 0x55dbb562ee20 crc :-1 s=READY pgs=205 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2033204773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5608c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 >> 172.21.15.90:0/2033204773 conn(0x55dbb5560720 msgr2=0x55dbb55fbca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f1861ffb700 0 client.17406 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:11.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 shutdown_connections 2024-04-06T14:17:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 1 -- 172.21.15.90:0/2033204773 wait complete. 2024-04-06T14:17:11.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.753+0000 7f188cc69b80 20 client.17406 trim_cache size 0 max 16384 2024-04-06T14:17:11.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.778+0000 7f188cc69b80 1 Processor -- start 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 -- start start 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5608be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5623b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb55f43b0 con 0x55dbb5601800 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb56240b0 con 0x55dbb560be60 2024-04-06T14:17:11.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f188cc69b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb5602500 con 0x55dbb5602120 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f187ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f187ad9d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59242/0 (socket says 172.21.15.90:59242) 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.782+0000 7f187ad9d700 1 -- 172.21.15.90:0/2795824297 learned_addr learned my addr 172.21.15.90:0/2795824297 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 -- 172.21.15.90:0/2795824297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 msgr2=0x55dbb5623b70 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5623b70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 -- 172.21.15.90:0/2795824297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 msgr2=0x55dbb5608be0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5608be0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 -- 172.21.15.90:0/2795824297 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dbb5633a50 con 0x55dbb560be60 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f187ad9d700 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f187409ad40 tx=0x7f1874097400 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cac254208d3c535d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f18697fa700 1 -- 172.21.15.90:0/2795824297 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18740b2070 con 0x55dbb560be60 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f18697fa700 1 -- 172.21.15.90:0/2795824297 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1874019350 con 0x55dbb560be60 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.783+0000 7f18697fa700 1 -- 172.21.15.90:0/2795824297 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18740993e0 con 0x55dbb560be60 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- 172.21.15.90:0/2795824297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 msgr2=0x55dbb560b3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7f187409ad40 tx=0x7f1874097400 comp rx=0 tx=0).stop 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- 172.21.15.90:0/2795824297 shutdown_connections 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb560be60 0x55dbb560b3e0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5601800 0x55dbb5608be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2795824297 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb5602120 0x55dbb5623b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- 172.21.15.90:0/2795824297 >> 172.21.15.90:0/2795824297 conn(0x55dbb5560720 msgr2=0x55dbb56084d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- 172.21.15.90:0/2795824297 shutdown_connections 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- 172.21.15.90:0/2795824297 wait complete. 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- start start 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 Processor -- start 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5602120 0x55dbb5622ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb560be60 0x55dbb5680840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb5680d80 con 0x55dbb5602120 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb5680ef0 con 0x55dbb5601800 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.784+0000 7f188cc69b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dbb56810b0 con 0x55dbb560be60 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f188cc69b80 10 client.0 _ll_register_callbacks cb 0x55dbb54f84f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59258/0 (socket says 172.21.15.90:59258) 2024-04-06T14:17:11.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 -- 172.21.15.90:0/2580749947 learned_addr learned my addr 172.21.15.90:0/2580749947 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f1879d9b700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5602120 0x55dbb5622ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 -- 172.21.15.90:0/2580749947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb560be60 msgr2=0x55dbb5680840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb560be60 0x55dbb5680840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 -- 172.21.15.90:0/2580749947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5602120 msgr2=0x55dbb5622ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5602120 0x55dbb5622ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 -- 172.21.15.90:0/2580749947 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f187c06a9b0 con 0x55dbb5601800 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f187a59c700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f1870009710 tx=0x7f187000b820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=539296a3ab769215 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f1861ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f187002db20 con 0x55dbb5601800 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1870011e50 con 0x55dbb5601800 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f1870000e40 con 0x55dbb5601800 2024-04-06T14:17:11.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f187002d630 con 0x55dbb5601800 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f188cc69b80 10 client.27137 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.785+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dbb5518f20 con 0x55dbb5601800 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1870011e50 con 0x55dbb5601800 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f18700117c0 con 0x55dbb5601800 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 10 client.27137 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:11.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 10 client.27137 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 20 client.27137 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb5682530 0x55dbb5684950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 4 client.27137 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55dbb53a1d70 con 0x55dbb5682530 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f1879d9b700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb5682530 0x55dbb5684950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:11.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.786+0000 7f1861ffb700 10 client.27137 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:11.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.787+0000 7f1879d9b700 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb5682530 0x55dbb5684950 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4d550655dcd32cc8 server_cookie=f1566584f8f3035 in_seq=0 out_seq=0 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.787+0000 7f1861ffb700 1 -- 172.21.15.90:0/2580749947 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55dbb53a1d70 con 0x55dbb5682530 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.787+0000 7f1861ffb700 10 client.27137 handle_command_reply: tid=0 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.787+0000 7f188cc69b80 1 client.27137 shutdown 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.788+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 msgr2=0x55dbb5622990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.788+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7f1870009710 tx=0x7f187000b820 comp rx=0 tx=0).stop 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 shutdown_connections 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55dbb5682530 0x55dbb5684950 crc :-1 s=READY pgs=206 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dbb560be60 0x55dbb5680840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dbb5601800 0x55dbb5622990 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 --2- 172.21.15.90:0/2580749947 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dbb5602120 0x55dbb5622ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 >> 172.21.15.90:0/2580749947 conn(0x55dbb5560720 msgr2=0x55dbb56007f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.791+0000 7f1861ffb700 0 client.27137 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.792+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 shutdown_connections 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.792+0000 7f188cc69b80 1 -- 172.21.15.90:0/2580749947 wait complete. 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.795+0000 7f188cc69b80 20 client.27137 trim_cache size 0 max 16384 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "40e01e6d-4b0a-473e-8e3f-44cdd942ca41", 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:17:11.796 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.796+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 msgr2=0x7f18640a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.796+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 0x7f18640a3910 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x55dbb5603270 tx=0x7f186c008040 comp rx=0 tx=0).stop 2024-04-06T14:17:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.796+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 msgr2=0x7f187c19da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:11.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.796+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 0x7f187c19da80 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f187400b0a0 tx=0x7f18740077f0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 shutdown_connections 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f18640a1450 0x7f18640a3910 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f187c105e60 0x7f187c19da80 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f187c0ff220 0x7f187c1a3950 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 --2- 172.21.15.90:0/626403093 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f187c0ffcc0 0x7f187c1a3e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 >> 172.21.15.90:0/626403093 conn(0x7f187c0761b0 msgr2=0x7f187c101540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 shutdown_connections 2024-04-06T14:17:11.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:11.799+0000 7f1880ab5700 1 -- 172.21.15.90:0/626403093 wait complete. 2024-04-06T14:17:11.818 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:17:11.818 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:17:12.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 -- 172.21.15.90:0/1399331628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd14411a590 msgr2=0x7fd14411c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/1399331628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd14411a590 0x7fd14411c980 secure :-1 s=READY pgs=773 cs=0 l=1 rev1=1 crypto rx=0x7fd13c01c0d0 tx=0x7fd13c01c3e0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 -- 172.21.15.90:0/1399331628 shutdown_connections 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/1399331628 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd14411a590 0x7fd14411c980 unknown :-1 s=CLOSED pgs=773 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/1399331628 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14410f4b0 0x7fd144107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/1399331628 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd14410f920 0x7fd1441082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.096+0000 7fd14b7bb700 1 -- 172.21.15.90:0/1399331628 >> 172.21.15.90:0/1399331628 conn(0x7fd14406be00 msgr2=0x7fd14406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.101+0000 7fd14b7bb700 1 -- 172.21.15.90:0/1399331628 shutdown_connections 2024-04-06T14:17:12.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.108+0000 7fd14b7bb700 1 -- 172.21.15.90:0/1399331628 wait complete. 2024-04-06T14:17:12.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.108+0000 7fd14b7bb700 1 Processor -- start 2024-04-06T14:17:12.111 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 -- start start 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd144115d80 0x7fd144116200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd144116740 0x7fd1441aa220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd14411a320 con 0x7fd144116740 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1441aa760 con 0x7fd14411a590 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.111+0000 7fd14b7bb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1441aa8a0 con 0x7fd144115d80 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59262/0 (socket says 172.21.15.90:59262) 2024-04-06T14:17:12.112 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 -- 172.21.15.90:0/2408252024 learned_addr learned my addr 172.21.15.90:0/2408252024 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149d58700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd144116740 0x7fd1441aa220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 -- 172.21.15.90:0/2408252024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd144115d80 msgr2=0x7fd144116200 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd144115d80 0x7fd144116200 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 -- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd144116740 msgr2=0x7fd1441aa220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd144116740 0x7fd1441aa220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.112+0000 7fd149557700 1 -- 172.21.15.90:0/2408252024 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd13c01ad10 con 0x7fd14411a590 2024-04-06T14:17:12.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd149557700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fd13400b450 tx=0x7fd13400b760 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd13a7fc700 1 -- 172.21.15.90:0/2408252024 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd134011630 con 0x7fd14411a590 2024-04-06T14:17:12.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd13a7fc700 1 -- 172.21.15.90:0/2408252024 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd134011c50 con 0x7fd14411a590 2024-04-06T14:17:12.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd13a7fc700 1 -- 172.21.15.90:0/2408252024 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd134010c20 con 0x7fd14411a590 2024-04-06T14:17:12.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1441aab00 con 0x7fd14411a590 2024-04-06T14:17:12.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.115+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1441aafc0 con 0x7fd14411a590 2024-04-06T14:17:12.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.121+0000 7fd13a7fc700 1 -- 172.21.15.90:0/2408252024 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fd134010da0 con 0x7fd14411a590 2024-04-06T14:17:12.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.122+0000 7fd13a7fc700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 0x7fd1300a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.122+0000 7fd13a7fc700 1 -- 172.21.15.90:0/2408252024 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fd1340cc290 con 0x7fd14411a590 2024-04-06T14:17:12.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.123+0000 7fd15796fb80 1 Processor -- start 2024-04-06T14:17:12.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 -- start start 2024-04-06T14:17:12.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11245780 0x55ad11245b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad112521d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad1119d990 con 0x55ad1124fde0 2024-04-06T14:17:12.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad1119b350 con 0x55ad112460a0 2024-04-06T14:17:12.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd15796fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad11196c40 con 0x55ad11245780 2024-04-06T14:17:12.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.125+0000 7fd148d56700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 0x7fd1300a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59266/0 (socket says 172.21.15.90:59266) 2024-04-06T14:17:12.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 -- 172.21.15.90:0/1181108112 learned_addr learned my addr 172.21.15.90:0/1181108112 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 -- 172.21.15.90:0/1181108112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11245780 msgr2=0x55ad11245b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11245780 0x55ad11245b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 -- 172.21.15.90:0/1181108112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 msgr2=0x55ad112521d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad112521d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 -- 172.21.15.90:0/1181108112 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad110183b0 con 0x55ad112460a0 2024-04-06T14:17:12.131 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.129+0000 7fd148d56700 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fd140014bd0 tx=0x7fd140017000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ebc51c2a0467a7d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.133+0000 7fd12f7fe700 1 -- 172.21.15.90:0/1181108112 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd140016e20 con 0x55ad112460a0 2024-04-06T14:17:12.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.133+0000 7fd148d56700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 0x7fd1300a38e0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fd140014480 tx=0x7fd140009330 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.133+0000 7fd12f7fe700 1 -- 172.21.15.90:0/1181108112 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd1400178d0 con 0x55ad112460a0 2024-04-06T14:17:12.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.133+0000 7fd12f7fe700 1 -- 172.21.15.90:0/1181108112 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd140021660 con 0x55ad112460a0 2024-04-06T14:17:12.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.136+0000 7fd15796fb80 1 -- 172.21.15.90:0/1181108112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 msgr2=0x55ad1124f8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.136+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7fd140014bd0 tx=0x7fd140017000 comp rx=0 tx=0).stop 2024-04-06T14:17:12.140 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.139+0000 7fd15796fb80 1 -- 172.21.15.90:0/1181108112 shutdown_connections 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.139+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad112521d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.139+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11245780 0x55ad11245b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.139+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1181108112 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad112460a0 0x55ad1124f8a0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.139+0000 7fd15796fb80 1 -- 172.21.15.90:0/1181108112 >> 172.21.15.90:0/1181108112 conn(0x55ad111a46a0 msgr2=0x55ad11240a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.140+0000 7fd15796fb80 1 -- 172.21.15.90:0/1181108112 shutdown_connections 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.140+0000 7fd15796fb80 1 -- 172.21.15.90:0/1181108112 wait complete. 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.140+0000 7fd15796fb80 1 -- start start 2024-04-06T14:17:12.141 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.140+0000 7fd15796fb80 1 Processor -- start 2024-04-06T14:17:12.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad1124f840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 0x55ad1124cb30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad11132d80 con 0x55ad112460a0 2024-04-06T14:17:12.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad11132a20 con 0x55ad11245780 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad1101a6a0 con 0x55ad1124fde0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd15796fb80 10 client.0 _ll_register_callbacks cb 0x55ad1113c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40150/0 (socket says 172.21.15.90:40150) 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 -- 172.21.15.90:0/7981661 learned_addr learned my addr 172.21.15.90:0/7981661 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149557700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad1124f840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 -- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 msgr2=0x55ad1124f840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad1124f840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd149d58700 1 -- 172.21.15.90:0/7981661 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 msgr2=0x55ad1124cb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.141+0000 7fd148d56700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 0x55ad1124cb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd149d58700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 0x55ad1124cb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd149d58700 1 -- 172.21.15.90:0/7981661 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd140014de0 con 0x55ad1124fde0 2024-04-06T14:17:12.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd149557700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad1124f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd148d56700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 0x55ad1124cb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd149d58700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fd13c01aa30 tx=0x7fd13c003ce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d582ae989721ed14 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd12cff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd13c01f070 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fd13c018900 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ad11157380 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd13c018ac0 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd15796fb80 10 client.26923 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.142+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad10fc4b60 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ad11157380 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fd13c034070 con 0x55ad1124fde0 2024-04-06T14:17:12.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 10 client.26923 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 10 client.26923 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 20 client.26923 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad1125fa70 0x55ad11261e90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 4 client.26923 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ad1118f700 con 0x55ad1125fa70 2024-04-06T14:17:12.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.143+0000 7fd149557700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad1125fa70 0x55ad11261e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.145+0000 7fd12cff9700 10 client.26923 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.145+0000 7fd149557700 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad1125fa70 0x55ad11261e90 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6ea9d53b6536cf75 server_cookie=feec3f22949d00be in_seq=0 out_seq=0 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.146+0000 7fd12cff9700 1 -- 172.21.15.90:0/7981661 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ad1118f700 con 0x55ad1125fa70 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.146+0000 7fd12cff9700 10 client.26923 handle_command_reply: tid=0 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.146+0000 7fd15796fb80 1 client.26923 shutdown 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.146+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 msgr2=0x55ad1124d270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.146+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fd13c01aa30 tx=0x7fd13c003ce0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 shutdown_connections 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad1125fa70 0x55ad11261e90 crc :-1 s=READY pgs=207 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad1124fde0 0x55ad1124d270 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad1124f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 --2- 172.21.15.90:0/7981661 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad112460a0 0x55ad1124cb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 >> 172.21.15.90:0/7981661 conn(0x55ad111a46a0 msgr2=0x55ad1123fc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd12cff9700 0 client.26923 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 shutdown_connections 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.147+0000 7fd15796fb80 1 -- 172.21.15.90:0/7981661 wait complete. 2024-04-06T14:17:12.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.148+0000 7fd15796fb80 20 client.26923 trim_cache size 0 max 16384 2024-04-06T14:17:12.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.169+0000 7fd15796fb80 1 Processor -- start 2024-04-06T14:17:12.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 -- start start 2024-04-06T14:17:12.171 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad11245780 0x55ad11256a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad1124fde0 0x55ad1124b940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad1123ad20 con 0x55ad11245780 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad1125c5c0 con 0x55ad1124fde0 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd15796fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad11247290 con 0x55ad112460a0 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd148d56700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd148d56700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40156/0 (socket says 172.21.15.90:40156) 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd148d56700 1 -- 172.21.15.90:0/3717527480 learned_addr learned my addr 172.21.15.90:0/3717527480 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.170+0000 7fd149d58700 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad1124fde0 0x55ad1124b940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 -- 172.21.15.90:0/3717527480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad1124fde0 msgr2=0x55ad1124b940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad1124fde0 0x55ad1124b940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 -- 172.21.15.90:0/3717527480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad11245780 msgr2=0x55ad11256a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad11245780 0x55ad11256a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 -- 172.21.15.90:0/3717527480 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ad1115cea0 con 0x55ad112460a0 2024-04-06T14:17:12.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd148d56700 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fd1400097f0 tx=0x7fd140006a90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=628bea3b154f9362 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd12ffff700 1 -- 172.21.15.90:0/3717527480 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd140049040 con 0x55ad112460a0 2024-04-06T14:17:12.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd12ffff700 1 -- 172.21.15.90:0/3717527480 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd140009910 con 0x55ad112460a0 2024-04-06T14:17:12.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.171+0000 7fd12ffff700 1 -- 172.21.15.90:0/3717527480 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd140017e30 con 0x55ad112460a0 2024-04-06T14:17:12.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 -- 172.21.15.90:0/3717527480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 msgr2=0x55ad1124b400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fd1400097f0 tx=0x7fd140006a90 comp rx=0 tx=0).stop 2024-04-06T14:17:12.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 -- 172.21.15.90:0/3717527480 shutdown_connections 2024-04-06T14:17:12.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad1124fde0 0x55ad1124b940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.174 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad11245780 0x55ad11256a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 --2- 172.21.15.90:0/3717527480 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad112460a0 0x55ad1124b400 secure :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fd1400097f0 tx=0x7fd140006a90 comp rx=0 tx=0).stop 2024-04-06T14:17:12.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.172+0000 7fd15796fb80 1 -- 172.21.15.90:0/3717527480 >> 172.21.15.90:0/3717527480 conn(0x55ad111a46a0 msgr2=0x55ad11247e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.173+0000 7fd15796fb80 1 -- 172.21.15.90:0/3717527480 shutdown_connections 2024-04-06T14:17:12.175 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.173+0000 7fd15796fb80 1 -- 172.21.15.90:0/3717527480 wait complete. 2024-04-06T14:17:12.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 -- start start 2024-04-06T14:17:12.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 Processor -- start 2024-04-06T14:17:12.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad11256990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.178 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad11254dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad112b3870 con 0x55ad1124fde0 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad112b39b0 con 0x55ad11245780 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.174+0000 7fd15796fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ad112b3b20 con 0x55ad11255310 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd15796fb80 10 client.0 _ll_register_callbacks cb 0x55ad1113c470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:40166/0 (socket says 172.21.15.90:40166) 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 -- 172.21.15.90:0/1986683191 learned_addr learned my addr 172.21.15.90:0/1986683191 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149557700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad11256990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 -- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 msgr2=0x55ad11256990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad11256990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.179 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 -- 172.21.15.90:0/1986683191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 msgr2=0x55ad11254dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad11254dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 -- 172.21.15.90:0/1986683191 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ad110183b0 con 0x55ad11255310 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149557700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad11256990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd149d58700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fd13c002230 tx=0x7fd13c003e70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=98cc8ff3d8cde0cf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd12cff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.175+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd13c01f070 con 0x55ad11255310 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fd13c023830 con 0x55ad11255310 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fd1400178d0 con 0x55ad11255310 2024-04-06T14:17:12.180 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd13c006070 con 0x55ad11255310 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 10 client.26929 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ad1101ccd0 con 0x55ad11255310 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd1400178d0 con 0x55ad11255310 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fd13c02a040 con 0x55ad11255310 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 10 client.26929 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 10 client.26929 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:12.181 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 20 client.26929 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad112b5020 0x55ad112b7440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 4 client.26929 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ad10fe5cf0 con 0x55ad112b5020 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.176+0000 7fd149557700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad112b5020 0x55ad112b7440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd12cff9700 10 client.26929 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd149557700 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad112b5020 0x55ad112b7440 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6c2b95b892386808 server_cookie=f5af0b811780246f in_seq=0 out_seq=0 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd12cff9700 1 -- 172.21.15.90:0/1986683191 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x55ad10fe5cf0 con 0x55ad112b5020 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd12cff9700 10 client.26929 handle_command_reply: tid=0 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 client.26929 shutdown 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 msgr2=0x55ad11255790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.182 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fd13c002230 tx=0x7fd13c003e70 comp rx=0 tx=0).stop 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 shutdown_connections 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ad112b5020 0x55ad112b7440 crc :-1 s=READY pgs=208 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ad11255310 0x55ad11255790 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ad11245780 0x55ad11256990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd12cff9700 0 client.26929 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 --2- 172.21.15.90:0/1986683191 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ad1124fde0 0x55ad11254dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 >> 172.21.15.90:0/1986683191 conn(0x55ad111a46a0 msgr2=0x55ad1124c070 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 shutdown_connections 2024-04-06T14:17:12.183 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.177+0000 7fd15796fb80 1 -- 172.21.15.90:0/1986683191 wait complete. 2024-04-06T14:17:12.184 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.178+0000 7fd15796fb80 20 client.26929 trim_cache size 0 max 16384 2024-04-06T14:17:12.184 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:12.184 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:17:12.184 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:17:12.184 INFO:teuthology.orchestra.run.smithi090.stdout: "40e01e6d-4b0a-473e-8e3f-44cdd942ca41": { 2024-04-06T14:17:12.185 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:17:12.185 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "40e01e6d-4b0a-473e-8e3f-44cdd942ca41", 2024-04-06T14:17:12.185 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:17:12.185 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:17:12.185 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:17:12.186 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:12.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 msgr2=0x7fd1300a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 0x7fd1300a38e0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fd140014480 tx=0x7fd140009330 comp rx=0 tx=0).stop 2024-04-06T14:17:12.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 msgr2=0x7fd144119cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:12.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7fd13400b450 tx=0x7fd13400b760 comp rx=0 tx=0).stop 2024-04-06T14:17:12.187 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 shutdown_connections 2024-04-06T14:17:12.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fd1300a1420 0x7fd1300a38e0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fd144116740 0x7fd1441aa220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd14411a590 0x7fd144119cc0 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 --2- 172.21.15.90:0/2408252024 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fd144115d80 0x7fd144116200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:12.188 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.179+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 >> 172.21.15.90:0/2408252024 conn(0x7fd14406be00 msgr2=0x7fd14410c110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:12.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.180+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 shutdown_connections 2024-04-06T14:17:12.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:12.184+0000 7fd14b7bb700 1 -- 172.21.15.90:0/2408252024 wait complete. 2024-04-06T14:17:12.198 INFO:tasks.cephfs.filesystem:scrub status for tag:40e01e6d-4b0a-473e-8e3f-44cdd942ca41 - {'path': '/', 'tag': '40e01e6d-4b0a-473e-8e3f-44cdd942ca41', 'options': 'recursive,force'} 2024-04-06T14:17:12.229 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:11 smithi090 ceph-mon[43420]: osdmap e95: 12 total, 12 up, 12 in 2024-04-06T14:17:12.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:11 smithi105 ceph-mon[46969]: osdmap e95: 12 total, 12 up, 12 in 2024-04-06T14:17:12.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:11 smithi134 ceph-mon[47549]: osdmap e95: 12 total, 12 up, 12 in 2024-04-06T14:17:13.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 685.0 s, 1263.3 tps, lat 79.917 ms stddev 123.702 2024-04-06T14:17:13.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:12 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:17:13.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:12 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:13.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:12 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:17:13.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:12 smithi105 ceph-mon[46969]: pgmap v368: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.6 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.70k op/s 2024-04-06T14:17:13.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:12 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:13.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:12 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:17:13.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:12 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:13.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:12 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:17:13.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:12 smithi090 ceph-mon[43420]: pgmap v368: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.6 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.70k op/s 2024-04-06T14:17:13.298 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:12 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:12 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:17:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:12 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:12 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:17:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:12 smithi134 ceph-mon[47549]: pgmap v368: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.6 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 19 MiB/s wr, 1.70k op/s 2024-04-06T14:17:13.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:12 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:15.292 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:15 smithi090 ceph-mon[43420]: pgmap v369: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 15 MiB/s wr, 1.33k op/s 2024-04-06T14:17:15.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:15 smithi134 ceph-mon[47549]: pgmap v369: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 15 MiB/s wr, 1.33k op/s 2024-04-06T14:17:15.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:15 smithi105 ceph-mon[46969]: pgmap v369: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 15 MiB/s wr, 1.33k op/s 2024-04-06T14:17:16.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:16 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:17:16.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:16 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:17:16.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:16 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:17:17.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:17 smithi090 ceph-mon[43420]: pgmap v370: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-04-06T14:17:17.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:17 smithi134 ceph-mon[47549]: pgmap v370: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-04-06T14:17:17.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:17 smithi105 ceph-mon[46969]: pgmap v370: 97 pgs: 4 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 83 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 12 MiB/s wr, 1.04k op/s 2024-04-06T14:17:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 690.0 s, 1998.4 tps, lat 49.758 ms stddev 62.377 2024-04-06T14:17:19.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:18 smithi105 ceph-mon[46969]: pgmap v371: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 8.5 MiB/s wr, 669 op/s 2024-04-06T14:17:19.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:18 smithi090 ceph-mon[43420]: pgmap v371: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 8.5 MiB/s wr, 669 op/s 2024-04-06T14:17:19.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:18 smithi134 ceph-mon[47549]: pgmap v371: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 8.5 MiB/s wr, 669 op/s 2024-04-06T14:17:21.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:20 smithi105 ceph-mon[46969]: pgmap v372: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 7.7 MiB/s wr, 654 op/s 2024-04-06T14:17:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:20 smithi090 ceph-mon[43420]: pgmap v372: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 7.7 MiB/s wr, 654 op/s 2024-04-06T14:17:21.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:20 smithi134 ceph-mon[47549]: pgmap v372: 97 pgs: 97 active+clean; 5.9 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 53 KiB/s rd, 7.7 MiB/s wr, 654 op/s 2024-04-06T14:17:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 695.0 s, 1507.8 tps, lat 65.084 ms stddev 111.454 2024-04-06T14:17:23.301 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:23 smithi090 ceph-mon[43420]: pgmap v373: 97 pgs: 97 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:17:23.301 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:23 smithi090 ceph-mon[43420]: osdmap e96: 12 total, 12 up, 12 in 2024-04-06T14:17:23.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:23 smithi134 ceph-mon[47549]: pgmap v373: 97 pgs: 97 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:17:23.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:23 smithi134 ceph-mon[47549]: osdmap e96: 12 total, 12 up, 12 in 2024-04-06T14:17:23.504 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:23 smithi105 ceph-mon[46969]: pgmap v373: 97 pgs: 97 active+clean; 6.0 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 89 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:17:23.504 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:23 smithi105 ceph-mon[46969]: osdmap e96: 12 total, 12 up, 12 in 2024-04-06T14:17:25.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:25 smithi134 ceph-mon[47549]: pgmap v375: 97 pgs: 97 active+clean; 6.1 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 21 MiB/s wr, 1.75k op/s 2024-04-06T14:17:25.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:25 smithi105 ceph-mon[46969]: pgmap v375: 97 pgs: 97 active+clean; 6.1 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 21 MiB/s wr, 1.75k op/s 2024-04-06T14:17:25.610 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:25 smithi090 ceph-mon[43420]: pgmap v375: 97 pgs: 97 active+clean; 6.1 GiB data, 19 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 21 MiB/s wr, 1.75k op/s 2024-04-06T14:17:27.401 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:27 smithi134 ceph-mon[47549]: pgmap v376: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 23 MiB/s wr, 1.88k op/s 2024-04-06T14:17:27.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:27 smithi105 ceph-mon[46969]: pgmap v376: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 23 MiB/s wr, 1.88k op/s 2024-04-06T14:17:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:27 smithi090 ceph-mon[43420]: pgmap v376: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 23 MiB/s wr, 1.88k op/s 2024-04-06T14:17:28.030 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 700.0 s, 950.9 tps, lat 106.926 ms stddev 135.894 2024-04-06T14:17:29.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:28 smithi105 ceph-mon[46969]: pgmap v377: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 25 MiB/s wr, 2.15k op/s 2024-04-06T14:17:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:28 smithi090 ceph-mon[43420]: pgmap v377: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 25 MiB/s wr, 2.15k op/s 2024-04-06T14:17:29.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:28 smithi134 ceph-mon[47549]: pgmap v377: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 25 MiB/s wr, 2.15k op/s 2024-04-06T14:17:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:31 smithi134 ceph-mon[47549]: pgmap v378: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 26 MiB/s wr, 2.23k op/s 2024-04-06T14:17:31.500 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:31 smithi105 ceph-mon[46969]: pgmap v378: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 26 MiB/s wr, 2.23k op/s 2024-04-06T14:17:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:31 smithi090 ceph-mon[43420]: pgmap v378: 97 pgs: 97 active+clean; 6.1 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 92 KiB/s rd, 26 MiB/s wr, 2.23k op/s 2024-04-06T14:17:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 705.0 s, 1289.3 tps, lat 77.889 ms stddev 85.854 2024-04-06T14:17:33.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:33 smithi134 ceph-mon[47549]: pgmap v379: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:33.509 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:33 smithi105 ceph-mon[46969]: pgmap v379: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:33.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:33 smithi090 ceph-mon[43420]: pgmap v379: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 20 MiB/s wr, 1.61k op/s 2024-04-06T14:17:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:35 smithi134 ceph-mon[47549]: pgmap v380: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 19 MiB/s wr, 1.40k op/s 2024-04-06T14:17:35.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:35 smithi105 ceph-mon[46969]: pgmap v380: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 19 MiB/s wr, 1.40k op/s 2024-04-06T14:17:35.551 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:35 smithi090 ceph-mon[43420]: pgmap v380: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 19 MiB/s wr, 1.40k op/s 2024-04-06T14:17:37.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:37 smithi134 ceph-mon[47549]: pgmap v381: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 18 MiB/s wr, 1.20k op/s 2024-04-06T14:17:37.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:37 smithi105 ceph-mon[46969]: pgmap v381: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 18 MiB/s wr, 1.20k op/s 2024-04-06T14:17:37.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:37 smithi090 ceph-mon[43420]: pgmap v381: 97 pgs: 97 active+clean; 6.2 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 18 MiB/s wr, 1.20k op/s 2024-04-06T14:17:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 710.0 s, 1566.2 tps, lat 64.301 ms stddev 68.856 2024-04-06T14:17:39.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:38 smithi105 ceph-mon[46969]: pgmap v382: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 19 MiB/s wr, 1.19k op/s 2024-04-06T14:17:39.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:38 smithi090 ceph-mon[43420]: pgmap v382: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 19 MiB/s wr, 1.19k op/s 2024-04-06T14:17:39.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:38 smithi134 ceph-mon[47549]: pgmap v382: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 19 MiB/s wr, 1.19k op/s 2024-04-06T14:17:40.256 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:39 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:17:40.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:39 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:17:40.395 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:39 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:17:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:40 smithi105 ceph-mon[46969]: pgmap v383: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 802 op/s 2024-04-06T14:17:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:40 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:17:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:40 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:17:41.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:40 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:17:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:40 smithi090 ceph-mon[43420]: pgmap v383: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 802 op/s 2024-04-06T14:17:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:40 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:17:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:40 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:17:41.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:40 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:17:41.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:40 smithi134 ceph-mon[47549]: pgmap v383: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 15 MiB/s wr, 802 op/s 2024-04-06T14:17:41.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:40 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:17:41.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:40 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:17:41.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:40 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:17:42.199 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:17:42.199 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:17:42.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- 172.21.15.90:0/1181445229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 msgr2=0x7f54e8107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/1181445229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e8107db0 secure :-1 s=READY pgs=774 cs=0 l=1 rev1=1 crypto rx=0x7f54d8009990 tx=0x7f54d8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- 172.21.15.90:0/1181445229 shutdown_connections 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/1181445229 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 0x7f54e811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/1181445229 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e8107db0 unknown :-1 s=CLOSED pgs=774 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/1181445229 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 0x7f54e81082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- 172.21.15.90:0/1181445229 >> 172.21.15.90:0/1181445229 conn(0x7f54e806be00 msgr2=0x7f54e806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- 172.21.15.90:0/1181445229 shutdown_connections 2024-04-06T14:17:42.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- 172.21.15.90:0/1181445229 wait complete. 2024-04-06T14:17:42.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 Processor -- start 2024-04-06T14:17:42.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.565+0000 7f54ee2f0700 1 -- start start 2024-04-06T14:17:42.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 0x7f54e81a6550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 0x7f54e81aa8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54e81aaf30 con 0x7f54e810f4b0 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54e81ab0a0 con 0x7f54e810f920 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ee2f0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f54e81ab260 con 0x7f54e811a590 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57356/0 (socket says 172.21.15.90:57356) 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 -- 172.21.15.90:0/2965026128 learned_addr learned my addr 172.21.15.90:0/2965026128 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e77fe700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 0x7f54e81a6550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54ec88d700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 0x7f54e81aa8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 -- 172.21.15.90:0/2965026128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 msgr2=0x7f54e81aa8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 0x7f54e81aa8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 -- 172.21.15.90:0/2965026128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 msgr2=0x7f54e81a6550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 0x7f54e81a6550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 -- 172.21.15.90:0/2965026128 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f54d80095f0 con 0x7f54e810f4b0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.566+0000 7f54e7fff700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f54d8000c00 tx=0x7f54d8004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.567+0000 7f54e57fa700 1 -- 172.21.15.90:0/2965026128 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54d801d070 con 0x7f54e810f4b0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.567+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f54e81ab4e0 con 0x7f54e810f4b0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.567+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54e81b2d80 con 0x7f54e810f4b0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.567+0000 7f54e57fa700 1 -- 172.21.15.90:0/2965026128 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f54d8022470 con 0x7f54e810f4b0 2024-04-06T14:17:42.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.567+0000 7f54e57fa700 1 -- 172.21.15.90:0/2965026128 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54d8017650 con 0x7f54e810f4b0 2024-04-06T14:17:42.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.572+0000 7f54fa4a4b80 1 Processor -- start 2024-04-06T14:17:42.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 -- start start 2024-04-06T14:17:42.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54e57fa700 1 -- 172.21.15.90:0/2965026128 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f54d800fac0 con 0x7f54e810f4b0 2024-04-06T14:17:42.573 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a68a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57360/0 (socket says 172.21.15.90:57360) 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a91d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581186f4990 con 0x55811879c780 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581186f2350 con 0x5581187a6de0 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581186edc40 con 0x55811879d0a0 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54ec88d700 1 -- 172.21.15.90:0/4220845869 learned_addr learned my addr 172.21.15.90:0/4220845869 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.575 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.573+0000 7f54e7fff700 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a68a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 -- 172.21.15.90:0/4220845869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 msgr2=0x5581187a68a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a68a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 -- 172.21.15.90:0/4220845869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 msgr2=0x5581187a91d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a91d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 -- 172.21.15.90:0/4220845869 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55811856f3b0 con 0x55811879c780 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54e7fff700 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a68a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54e57fa700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 0x7f54d00a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f54e0003f40 tx=0x7f54e0009ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f4ba664f203fcd00 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54e7fff700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 0x7f54d00a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ce7fc700 1 -- 172.21.15.90:0/4220845869 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54e0019070 con 0x55811879c780 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54e57fa700 1 -- 172.21.15.90:0/2965026128 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f54d80cdde0 con 0x7f54e810f4b0 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ce7fc700 1 -- 172.21.15.90:0/4220845869 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f54e000f040 con 0x55811879c780 2024-04-06T14:17:42.576 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.574+0000 7f54ce7fc700 1 -- 172.21.15.90:0/4220845869 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54e0007a60 con 0x55811879c780 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4220845869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 msgr2=0x55811879cb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54e7fff700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 0x7f54d00a3850 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f54d809ae90 tx=0x7f54d80d17d0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 secure :-1 s=READY pgs=776 cs=0 l=1 rev1=1 crypto rx=0x7f54e0003f40 tx=0x7f54e0009ba0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4220845869 shutdown_connections 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a91d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x55811879cb60 unknown :-1 s=CLOSED pgs=776 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4220845869 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a68a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4220845869 >> 172.21.15.90:0/4220845869 conn(0x5581186fb6a0 msgr2=0x558118797a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4220845869 shutdown_connections 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.575+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4220845869 wait complete. 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 -- start start 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 Processor -- start 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187a67d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a41d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558118689d80 con 0x55811879c780 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558118689a20 con 0x5581187a6de0 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581185716a0 con 0x55811879d0a0 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54fa4a4b80 10 client.0 _ll_register_callbacks cb 0x558118693470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:42.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54e77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a41d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54e77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54308/0 (socket says 172.21.15.90:54308) 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.576+0000 7f54e77fe700 1 -- 172.21.15.90:0/82952475 learned_addr learned my addr 172.21.15.90:0/82952475 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 -- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 msgr2=0x5581187a41d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a41d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 -- 172.21.15.90:0/82952475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 msgr2=0x5581187a67d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187a67d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 -- 172.21.15.90:0/82952475 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54e81b2d80 con 0x55811879d0a0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54e77fe700 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f54dc0116c0 tx=0x7f54dc0119d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=99ad4fa41bdc9c3b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54c77fe700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54dc00e680 con 0x55811879d0a0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5581186ae380 con 0x55811879d0a0 2024-04-06T14:17:42.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f54dc00e7e0 con 0x55811879d0a0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f54dc00e9a0 con 0x55811879d0a0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54fa4a4b80 10 client.26941 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f54e0003b70 con 0x55811879d0a0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.577+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f54dc026020 con 0x55811879d0a0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f54dc021b40 con 0x55811879d0a0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 10 client.26941 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 10 client.26941 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 20 client.26941 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581187b6a90 0x5581187b8eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 4 client.26941 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5581186e6700 con 0x5581187b6a90 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54ec88d700 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581187b6a90 0x5581187b8eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54c77fe700 10 client.26941 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.578+0000 7f54ec88d700 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581187b6a90 0x5581187b8eb0 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4654c86d3713053f server_cookie=83e877aa967129a2 in_seq=0 out_seq=0 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54c77fe700 1 -- 172.21.15.90:0/82952475 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5581186e6700 con 0x5581187b6a90 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54c77fe700 10 client.26941 handle_command_reply: tid=0 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 client.26941 shutdown 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 msgr2=0x5581187a3bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f54dc0116c0 tx=0x7f54dc0119d0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 shutdown_connections 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5581187a6de0 0x5581187a41d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187a67d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5581187b6a90 0x5581187b8eb0 crc :-1 s=READY pgs=209 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/82952475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55811879d0a0 0x5581187a3bc0 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54c77fe700 0 client.26941 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:42.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 >> 172.21.15.90:0/82952475 conn(0x5581186fb6a0 msgr2=0x558118796c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 shutdown_connections 2024-04-06T14:17:42.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/82952475 wait complete. 2024-04-06T14:17:42.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.579+0000 7f54fa4a4b80 20 client.26941 trim_cache size 0 max 16384 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 Processor -- start 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 -- start start 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187a2490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187a29d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581187ae050 con 0x55811879c780 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581187a3990 con 0x55811879d0a0 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5581187a47b0 con 0x5581187a6de0 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.600 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57386/0 (socket says 172.21.15.90:57386) 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54ec88d700 1 -- 172.21.15.90:0/2336547668 learned_addr learned my addr 172.21.15.90:0/2336547668 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54e77fe700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187a2490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.598+0000 7f54ec88d700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187a29d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 -- 172.21.15.90:0/2336547668 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 msgr2=0x5581187a29d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187a29d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 -- 172.21.15.90:0/2336547668 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 msgr2=0x5581187a2490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187a2490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 -- 172.21.15.90:0/2336547668 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55811851bb60 con 0x55811879c780 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54e77fe700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187a2490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ec88d700 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f54e0003f40 tx=0x7f54e0007640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4aeb6cbda0b3bab9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ceffd700 1 -- 172.21.15.90:0/2336547668 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54e0019040 con 0x55811879c780 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ceffd700 1 -- 172.21.15.90:0/2336547668 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f54e0023070 con 0x55811879c780 2024-04-06T14:17:42.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.599+0000 7f54ceffd700 1 -- 172.21.15.90:0/2336547668 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54e00076b0 con 0x55811879c780 2024-04-06T14:17:42.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/2336547668 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 msgr2=0x5581187adaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 secure :-1 s=READY pgs=777 cs=0 l=1 rev1=1 crypto rx=0x7f54e0003f40 tx=0x7f54e0007640 comp rx=0 tx=0).stop 2024-04-06T14:17:42.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/2336547668 shutdown_connections 2024-04-06T14:17:42.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187a29d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187adaf0 unknown :-1 s=CLOSED pgs=777 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/2336547668 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187a2490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/2336547668 >> 172.21.15.90:0/2336547668 conn(0x5581186fb6a0 msgr2=0x55811879eee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/2336547668 shutdown_connections 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/2336547668 wait complete. 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 -- start start 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.600+0000 7f54fa4a4b80 1 Processor -- start 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187ac610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187acb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55811880a8d0 con 0x55811879c780 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55811880aa10 con 0x55811879d0a0 2024-04-06T14:17:42.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55811880ab50 con 0x5581187a6de0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54fa4a4b80 10 client.0 _ll_register_callbacks cb 0x558118693470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54ec88d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57390/0 (socket says 172.21.15.90:57390) 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54ec88d700 1 -- 172.21.15.90:0/4277750676 learned_addr learned my addr 172.21.15.90:0/4277750676 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187acb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.601+0000 7f54e77fe700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187ac610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 -- 172.21.15.90:0/4277750676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 msgr2=0x5581187acb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187acb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 -- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 msgr2=0x5581187ac610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187ac610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 -- 172.21.15.90:0/4277750676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54e0003b70 con 0x55811879c780 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54e77fe700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187ac610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54ec88d700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f54e0009180 tx=0x7f54e0009380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d63fa242fe709c9f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54c77fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f54e0019070 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f54e0008740 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.602+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f54e000f040 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f54e0008900 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 10 client.17430 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55811856f3b0 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f54e000f040 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f54e0025830 con 0x55811879c780 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 10 client.17430 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 10 client.17430 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 20 client.17430 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55811880be50 0x55811880e240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 4 client.17430 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55811853ccf0 con 0x55811880be50 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.603+0000 7f54e77fe700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55811880be50 0x55811880e240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54c77fe700 10 client.17430 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54e77fe700 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55811880be50 0x55811880e240 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=84d16f739094a5ab server_cookie=7183fcb8da7b7529 in_seq=0 out_seq=0 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54c77fe700 1 -- 172.21.15.90:0/4277750676 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55811853ccf0 con 0x55811880be50 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54c77fe700 10 client.17430 handle_command_reply: tid=0 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 client.17430 shutdown 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 msgr2=0x5581187ac0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 secure :-1 s=READY pgs=778 cs=0 l=1 rev1=1 crypto rx=0x7f54e0009180 tx=0x7f54e0009380 comp rx=0 tx=0).stop 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 shutdown_connections 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55811880be50 0x55811880e240 crc :-1 s=READY pgs=210 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5581187a6de0 0x5581187acb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55811879c780 0x5581187ac0d0 unknown :-1 s=CLOSED pgs=778 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 --2- 172.21.15.90:0/4277750676 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55811879d0a0 0x5581187ac610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 >> 172.21.15.90:0/4277750676 conn(0x5581186fb6a0 msgr2=0x55811879eee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54c77fe700 0 client.17430 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 shutdown_connections 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 1 -- 172.21.15.90:0/4277750676 wait complete. 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.604+0000 7f54fa4a4b80 20 client.17430 trim_cache size 0 max 16384 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:17:42.610 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:42.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 msgr2=0x7f54d00a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 0x7f54d00a3850 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f54d809ae90 tx=0x7f54d80d17d0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 msgr2=0x7f54e81a6010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 secure :-1 s=READY pgs=775 cs=0 l=1 rev1=1 crypto rx=0x7f54d8000c00 tx=0x7f54d8004730 comp rx=0 tx=0).stop 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 shutdown_connections 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f54d00a1390 0x7f54d00a3850 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f54e811a590 0x7f54e81aa8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f54e810f4b0 0x7f54e81a6010 unknown :-1 s=CLOSED pgs=775 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 --2- 172.21.15.90:0/2965026128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f54e810f920 0x7f54e81a6550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 >> 172.21.15.90:0/2965026128 conn(0x7f54e806be00 msgr2=0x7f54e8118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 shutdown_connections 2024-04-06T14:17:42.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.605+0000 7f54ee2f0700 1 -- 172.21.15.90:0/2965026128 wait complete. 2024-04-06T14:17:42.640 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:17:42.640 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/1019504751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0100160 msgr2=0x7fc1c0103f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/1019504751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0100160 0x7fc1c0103f20 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac009990 tx=0x7fc1ac009ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/1019504751 shutdown_connections 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/1019504751 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c0104650 0x7fc1c010c1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/1019504751 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c00ffb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/1019504751 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0100160 0x7fc1c0103f20 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/1019504751 >> 172.21.15.90:0/1019504751 conn(0x7fc1c0074e50 msgr2=0x7fc1c0075260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/1019504751 shutdown_connections 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.801+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/1019504751 wait complete. 2024-04-06T14:17:42.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 Processor -- start 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 -- start start 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 0x7fc1c01984f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0198a80 0x7fc1c01a3030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1c01a3740 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1c01a38b0 con 0x7fc1c0104650 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c6d7e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1c01a3a70 con 0x7fc1c0198a80 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:57402/0 (socket says 172.21.15.90:57402) 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1bffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 0x7fc1c01984f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 -- 172.21.15.90:0/2124893855 learned_addr learned my addr 172.21.15.90:0/2124893855 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c531b700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0198a80 0x7fc1c01a3030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 -- 172.21.15.90:0/2124893855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0198a80 msgr2=0x7fc1c01a3030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0198a80 0x7fc1c01a3030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 -- 172.21.15.90:0/2124893855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 msgr2=0x7fc1c01984f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 0x7fc1c01984f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1c4b1a700 1 -- 172.21.15.90:0/2124893855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1ac0095f0 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.802+0000 7fc1bffff700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 0x7fc1c01984f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1c4b1a700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7fc1b400e9a0 tx=0x7fc1b400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1b4009790 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc1c01a3d50 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc1b4010430 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc1b4018680 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.803+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc1c01a41d0 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.805+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc1c0100a00 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.806+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc1b40105a0 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.807+0000 7fc1bdffb700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 0x7fc1b00a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.807+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc1b4014070 con 0x7fc1c00ff7b0 2024-04-06T14:17:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.807+0000 7fc1bffff700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 0x7fc1b00a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:42.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.808+0000 7fc1bffff700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 0x7fc1b00a3750 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac00b2e0 tx=0x7fc1ac005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:42.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:42.828+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fc1b4052ab0 con 0x7fc1c00ff7b0 2024-04-06T14:17:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 715.0 s, 1220.2 tps, lat 80.240 ms stddev 102.385 2024-04-06T14:17:43.185 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.185+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc1c00fda00 con 0x7fc1c00ff7b0 2024-04-06T14:17:43.186 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.186+0000 7fc1bdffb700 1 -- 172.21.15.90:0/2124893855 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fc1b4096300 con 0x7fc1c00ff7b0 2024-04-06T14:17:43.186 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:17:43.186 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 msgr2=0x7fc1b00a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 0x7fc1b00a3750 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac00b2e0 tx=0x7fc1ac005960 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 msgr2=0x7fc1c0197fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 secure :-1 s=READY pgs=779 cs=0 l=1 rev1=1 crypto rx=0x7fc1b400e9a0 tx=0x7fc1b400ecb0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 shutdown_connections 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc1b00a1290 0x7fc1b00a3750 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc1c0198a80 0x7fc1c01a3030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc1c00ff7b0 0x7fc1c0197fb0 unknown :-1 s=CLOSED pgs=779 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 --2- 172.21.15.90:0/2124893855 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc1c0104650 0x7fc1c01984f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 >> 172.21.15.90:0/2124893855 conn(0x7fc1c0074e50 msgr2=0x7fc1c00fe510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 shutdown_connections 2024-04-06T14:17:43.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.189+0000 7fc1c6d7e700 1 -- 172.21.15.90:0/2124893855 wait complete. 2024-04-06T14:17:43.190 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:17:43.205 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:43.205 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:17:43.345 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:43 smithi090 ceph-mon[43420]: pgmap v384: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 948 op/s 2024-04-06T14:17:43.411 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:43 smithi134 ceph-mon[47549]: pgmap v384: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 948 op/s 2024-04-06T14:17:43.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 -- 172.21.15.90:0/1872687994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c1009d0 msgr2=0x7f7a1c100db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 --2- 172.21.15.90:0/1872687994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c100db0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f7a1800fb60 tx=0x7f7a1800fe70 comp rx=0 tx=0).stop 2024-04-06T14:17:43.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 -- 172.21.15.90:0/1872687994 shutdown_connections 2024-04-06T14:17:43.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 --2- 172.21.15.90:0/1872687994 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a1c104fe0 0x7f7a1c10cb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 --2- 172.21.15.90:0/1872687994 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c100db0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 --2- 172.21.15.90:0/1872687994 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c101380 0x7f7a1c1048b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 -- 172.21.15.90:0/1872687994 >> 172.21.15.90:0/1872687994 conn(0x7f7a1c074f30 msgr2=0x7f7a1c075340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 -- 172.21.15.90:0/1872687994 shutdown_connections 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.518+0000 7f7a23356700 1 -- 172.21.15.90:0/1872687994 wait complete. 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 Processor -- start 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 -- start start 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c195050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a1c104fe0 0x7f7a1c195590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a1c19a3b0 con 0x7f7a1c1009d0 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a1c19a520 con 0x7f7a1c199930 2024-04-06T14:17:43.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a23356700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a1c19a6e0 con 0x7f7a1c104fe0 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a218f3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c195050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a218f3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34168/0 (socket says 172.21.15.90:34168) 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.519+0000 7f7a218f3700 1 -- 172.21.15.90:0/1215325007 learned_addr learned my addr 172.21.15.90:0/1215325007 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a210f2700 1 -- 172.21.15.90:0/1215325007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a1c104fe0 msgr2=0x7f7a1c195590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a210f2700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a1c104fe0 0x7f7a1c195590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a210f2700 1 -- 172.21.15.90:0/1215325007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 msgr2=0x7f7a1c199db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a210f2700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a210f2700 1 -- 172.21.15.90:0/1215325007 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a1800f790 con 0x7f7a1c1009d0 2024-04-06T14:17:43.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.520+0000 7f7a218f3700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:43.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a210f2700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c195050 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f7a1800fa80 tx=0x7f7a1800f430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a127fc700 1 -- 172.21.15.90:0/1215325007 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a18024070 con 0x7f7a1c1009d0 2024-04-06T14:17:43.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a127fc700 1 -- 172.21.15.90:0/1215325007 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a18005580 con 0x7f7a1c1009d0 2024-04-06T14:17:43.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a1c1a1be0 con 0x7f7a1c1009d0 2024-04-06T14:17:43.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a1c1a2060 con 0x7f7a1c1009d0 2024-04-06T14:17:43.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.521+0000 7f7a127fc700 1 -- 172.21.15.90:0/1215325007 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a18003920 con 0x7f7a1c1009d0 2024-04-06T14:17:43.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.524+0000 7f7a127fc700 1 -- 172.21.15.90:0/1215325007 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f7a1801e040 con 0x7f7a1c1009d0 2024-04-06T14:17:43.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.525+0000 7f7a127fc700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 0x7f7a080a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.525+0000 7f7a127fc700 1 -- 172.21.15.90:0/1215325007 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7a180d1a60 con 0x7f7a1c1009d0 2024-04-06T14:17:43.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.525+0000 7f7a208f1700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 0x7f7a080a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 Processor -- start 2024-04-06T14:17:43.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 -- start start 2024-04-06T14:17:43.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a208f1700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 0x7f7a080a37d0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c005950 tx=0x7f7a0c0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e74b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e7e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28e811d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28dcc990 con 0x55ab28e74780 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28dca350 con 0x55ab28e750a0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28dc5c40 con 0x55ab28e7ede0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e74b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e74b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58668/0 (socket says 172.21.15.90:58668) 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a210f2700 1 -- 172.21.15.90:0/376210479 learned_addr learned my addr 172.21.15.90:0/376210479 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.526+0000 7f7a208f1700 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e7e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 -- 172.21.15.90:0/376210479 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 msgr2=0x55ab28e811d0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28e811d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 -- 172.21.15.90:0/376210479 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 msgr2=0x55ab28e74b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e74b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 -- 172.21.15.90:0/376210479 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ab28c473b0 con 0x55ab28e750a0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.527+0000 7f7a208f1700 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e7e8a0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f7a1c0734a0 tx=0x7f7a0c011000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bfab2a40fe1c4b95 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.528+0000 7f7a077fe700 1 -- 172.21.15.90:0/376210479 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a0c010890 con 0x55ab28e750a0 2024-04-06T14:17:43.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.528+0000 7f7a077fe700 1 -- 172.21.15.90:0/376210479 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a0c02d430 con 0x55ab28e750a0 2024-04-06T14:17:43.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.528+0000 7f7a077fe700 1 -- 172.21.15.90:0/376210479 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a0c037670 con 0x55ab28e750a0 2024-04-06T14:17:43.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:43 smithi105 ceph-mon[46969]: pgmap v384: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 948 op/s 2024-04-06T14:17:43.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/376210479 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 msgr2=0x55ab28e7e8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.536 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e7e8a0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f7a1c0734a0 tx=0x7f7a0c011000 comp rx=0 tx=0).stop 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/376210479 shutdown_connections 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28e811d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e74b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/376210479 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e7e8a0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.529+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/376210479 >> 172.21.15.90:0/376210479 conn(0x55ab28dd36a0 msgr2=0x55ab28e6fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.530+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/376210479 shutdown_connections 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.530+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/376210479 wait complete. 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.530+0000 7f7a2f50ab80 1 -- start start 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.530+0000 7f7a2f50ab80 1 Processor -- start 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e74780 0x55ab28e7e870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 0x55ab28e7c360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28d61d80 con 0x55ab28e7ede0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28d61a20 con 0x55ab28e74780 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28c496a0 con 0x55ab28e750a0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a2f50ab80 10 client.0 _ll_register_callbacks cb 0x55ab28d6b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a218f3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 0x55ab28e7c360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e74780 0x55ab28e7e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51602/0 (socket says 172.21.15.90:51602) 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 -- 172.21.15.90:0/805464413 learned_addr learned my addr 172.21.15.90:0/805464413 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 -- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e74780 msgr2=0x55ab28e7e870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e74780 0x55ab28e7e870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 -- 172.21.15.90:0/805464413 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 msgr2=0x55ab28e7c360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 0x55ab28e7c360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 -- 172.21.15.90:0/805464413 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a0c00e7f0 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a218f3700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 0x55ab28e7c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a208f1700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c02da00 tx=0x7f7a0c005a90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b02755080be46e31 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.531+0000 7f7a04ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.534+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a0c03b040 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7a0c010ca0 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ab28d86380 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7a0c02db80 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 10 client.26959 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ab28bf3b60 con 0x55ab28e750a0 2024-04-06T14:17:43.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a0c04f020 con 0x55ab28e750a0 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7a0c03e070 con 0x55ab28e750a0 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 10 client.26959 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 10 client.26959 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 20 client.26959 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28e8ea40 0x55ab28e90e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 4 client.26959 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.535+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ab28dbe700 con 0x55ab28e8ea40 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.536+0000 7f7a210f2700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28e8ea40 0x55ab28e90e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.536+0000 7f7a04ff9700 10 client.26959 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.536+0000 7f7a210f2700 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28e8ea40 0x55ab28e90e60 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=2347965654a01e58 server_cookie=a7966227e5b590a8 in_seq=0 out_seq=0 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a04ff9700 1 -- 172.21.15.90:0/805464413 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ab28dbe700 con 0x55ab28e8ea40 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a04ff9700 10 client.26959 handle_command_reply: tid=0 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 client.26959 shutdown 2024-04-06T14:17:43.540 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 msgr2=0x55ab28e7bc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c02da00 tx=0x7f7a0c005a90 comp rx=0 tx=0).stop 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 shutdown_connections 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28e8ea40 0x55ab28e90e60 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e7ede0 0x55ab28e7c360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e74780 0x55ab28e7e870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/805464413 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7bc20 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.541 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.537+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 >> 172.21.15.90:0/805464413 conn(0x55ab28dd36a0 msgr2=0x55ab28e6ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.538+0000 7f7a04ff9700 0 client.26959 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:43.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.538+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 shutdown_connections 2024-04-06T14:17:43.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.538+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/805464413 wait complete. 2024-04-06T14:17:43.542 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.538+0000 7f7a2f50ab80 20 client.26959 trim_cache size 0 max 16384 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.555+0000 7f7a2f50ab80 1 Processor -- start 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.555+0000 7f7a2f50ab80 1 -- start start 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7b500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e7ede0 0x55ab28e7ba40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28e76ae0 con 0x55ab28e74780 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28e7bfb0 con 0x55ab28e7ede0 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28e759f0 con 0x55ab28e750a0 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58672/0 (socket says 172.21.15.90:58672) 2024-04-06T14:17:43.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 -- 172.21.15.90:0/3271630571 learned_addr learned my addr 172.21.15.90:0/3271630571 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a208f1700 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7b500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a218f3700 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e7ede0 0x55ab28e7ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 -- 172.21.15.90:0/3271630571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 msgr2=0x55ab28e7b500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7b500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 -- 172.21.15.90:0/3271630571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e7ede0 msgr2=0x55ab28e7ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e7ede0 0x55ab28e7ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 -- 172.21.15.90:0/3271630571 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ab28d8bea0 con 0x55ab28e74780 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.556+0000 7f7a210f2700 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f7a1809b6e0 tx=0x7f7a180d1040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17b716828b71a333 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a07fff700 1 -- 172.21.15.90:0/3271630571 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a1809ec70 con 0x55ab28e74780 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a07fff700 1 -- 172.21.15.90:0/3271630571 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a180a2aa0 con 0x55ab28e74780 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a07fff700 1 -- 172.21.15.90:0/3271630571 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a1809d0f0 con 0x55ab28e74780 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/3271630571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 msgr2=0x55ab28e8a950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 secure :-1 s=READY pgs=781 cs=0 l=1 rev1=1 crypto rx=0x7f7a1809b6e0 tx=0x7f7a180d1040 comp rx=0 tx=0).stop 2024-04-06T14:17:43.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/3271630571 shutdown_connections 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e7ede0 0x55ab28e7ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e8a950 unknown :-1 s=CLOSED pgs=781 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/3271630571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e750a0 0x55ab28e7b500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/3271630571 >> 172.21.15.90:0/3271630571 conn(0x55ab28dd36a0 msgr2=0x55ab28e8a240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/3271630571 shutdown_connections 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.557+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/3271630571 wait complete. 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 -- start start 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 Processor -- start 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e84010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28ee1830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28e848d0 con 0x55ab28e74780 2024-04-06T14:17:43.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28ee1d70 con 0x55ab28e750a0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ab28ee1eb0 con 0x55ab28e7ede0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a2f50ab80 10 client.0 _ll_register_callbacks cb 0x55ab28d6b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a210f2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58688/0 (socket says 172.21.15.90:58688) 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a210f2700 1 -- 172.21.15.90:0/2989118972 learned_addr learned my addr 172.21.15.90:0/2989118972 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a208f1700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e84010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.558+0000 7f7a218f3700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28ee1830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 -- 172.21.15.90:0/2989118972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 msgr2=0x55ab28ee1830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28ee1830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 -- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 msgr2=0x55ab28e84010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e84010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.563 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 -- 172.21.15.90:0/2989118972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a1c1a2060 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a210f2700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f7a1809e970 tx=0x7f7a180a7060 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5078765fe766b0e4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a1809ec70 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7a1809d010 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7a0c02d430 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7a18009b80 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 10 client.17451 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ab28c4bcd0 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a0c02d430 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7a180a6bd0 con 0x55ab28e74780 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 10 client.17451 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 10 client.17451 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 20 client.17451 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:43.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28ee3180 0x55ab28ee5570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 4 client.17451 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:43.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.559+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ab28c14cf0 con 0x55ab28ee3180 2024-04-06T14:17:43.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.560+0000 7f7a208f1700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28ee3180 0x55ab28ee5570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a208f1700 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28ee3180 0x55ab28ee5570 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=506fc66ef2db8231 server_cookie=a1fd34d4e53cb646 in_seq=0 out_seq=0 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a04ff9700 10 client.17451 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a04ff9700 1 -- 172.21.15.90:0/2989118972 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ab28c14cf0 con 0x55ab28ee3180 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a04ff9700 10 client.17451 handle_command_reply: tid=0 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a2f50ab80 1 client.17451 shutdown 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 msgr2=0x55ab28e83ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.561+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 secure :-1 s=READY pgs=782 cs=0 l=1 rev1=1 crypto rx=0x7f7a1809e970 tx=0x7f7a180a7060 comp rx=0 tx=0).stop 2024-04-06T14:17:43.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 shutdown_connections 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ab28ee3180 0x55ab28ee5570 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ab28e7ede0 0x55ab28ee1830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a04ff9700 0 client.17451 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ab28e74780 0x55ab28e83ad0 unknown :-1 s=CLOSED pgs=782 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 --2- 172.21.15.90:0/2989118972 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ab28e750a0 0x55ab28e84010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 >> 172.21.15.90:0/2989118972 conn(0x55ab28dd36a0 msgr2=0x55ab28e74000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 shutdown_connections 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 1 -- 172.21.15.90:0/2989118972 wait complete. 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.562+0000 7f7a2f50ab80 20 client.17451 trim_cache size 0 max 16384 2024-04-06T14:17:43.567 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:43.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 msgr2=0x7f7a080a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 0x7f7a080a37d0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f7a0c005950 tx=0x7f7a0c0058e0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 msgr2=0x7f7a1c195050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c195050 secure :-1 s=READY pgs=780 cs=0 l=1 rev1=1 crypto rx=0x7f7a1800fa80 tx=0x7f7a1800f430 comp rx=0 tx=0).stop 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 shutdown_connections 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7a080a1310 0x7f7a080a37d0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7a1c199930 0x7f7a1c199db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7a1c1009d0 0x7f7a1c195050 unknown :-1 s=CLOSED pgs=780 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 --2- 172.21.15.90:0/1215325007 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7a1c104fe0 0x7f7a1c195590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 >> 172.21.15.90:0/1215325007 conn(0x7f7a1c074f30 msgr2=0x7f7a1c0fda20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 shutdown_connections 2024-04-06T14:17:43.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.563+0000 7f7a23356700 1 -- 172.21.15.90:0/1215325007 wait complete. 2024-04-06T14:17:43.576 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:43.577 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 -- 172.21.15.90:0/4130153436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc101770 msgr2=0x7fafcc1057b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 --2- 172.21.15.90:0/4130153436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc101770 0x7fafcc1057b0 secure :-1 s=READY pgs=783 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009990 tx=0x7fafc4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 -- 172.21.15.90:0/4130153436 shutdown_connections 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 --2- 172.21.15.90:0/4130153436 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 0x7fafcc10da60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 --2- 172.21.15.90:0/4130153436 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc1011a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 --2- 172.21.15.90:0/4130153436 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc101770 0x7fafcc1057b0 unknown :-1 s=CLOSED pgs=783 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 -- 172.21.15.90:0/4130153436 >> 172.21.15.90:0/4130153436 conn(0x7fafcc0fc660 msgr2=0x7fafcc0fea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 -- 172.21.15.90:0/4130153436 shutdown_connections 2024-04-06T14:17:43.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.755+0000 7fafd359e700 1 -- 172.21.15.90:0/4130153436 wait complete. 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 Processor -- start 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 -- start start 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc197030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 0x7fafcc197570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafcc19c390 con 0x7fafcc19b910 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafcc19c500 con 0x7fafcc100dc0 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.756+0000 7fafd359e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafcc19c6c0 con 0x7fafcc105ee0 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58704/0 (socket says 172.21.15.90:58704) 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd259c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc197030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 -- 172.21.15.90:0/3409767546 learned_addr learned my addr 172.21.15.90:0/3409767546 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 0x7fafcc197570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 -- 172.21.15.90:0/3409767546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 msgr2=0x7fafcc197570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 0x7fafcc197570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 -- 172.21.15.90:0/3409767546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 msgr2=0x7fafcc197030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc197030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 -- 172.21.15.90:0/3409767546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fafc40095f0 con 0x7fafcc19b910 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd259c700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc197030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fafc800ef30 tx=0x7fafc800c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafbf7fe700 1 -- 172.21.15.90:0/3409767546 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc8018070 con 0x7fafcc19b910 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafbf7fe700 1 -- 172.21.15.90:0/3409767546 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fafc800f040 con 0x7fafcc19b910 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.757+0000 7fafbf7fe700 1 -- 172.21.15.90:0/3409767546 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc800cd00 con 0x7fafcc19b910 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.758+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fafcc1a3bc0 con 0x7fafcc19b910 2024-04-06T14:17:43.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.758+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafcc1a4050 con 0x7fafcc19b910 2024-04-06T14:17:43.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.761+0000 7fafbf7fe700 1 -- 172.21.15.90:0/3409767546 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fafc8003a00 con 0x7fafcc19b910 2024-04-06T14:17:43.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafbf7fe700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 0x7fafb80a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 Processor -- start 2024-04-06T14:17:43.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafbf7fe700 1 -- 172.21.15.90:0/3409767546 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fafc80cc7a0 con 0x7fafcc19b910 2024-04-06T14:17:43.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 -- start start 2024-04-06T14:17:43.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafd259c700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 0x7fafb80a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb71a3a0 0x560feb7187e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb72ea40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb671990 con 0x560feb718d20 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb66f350 con 0x560feb72c650 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb66ac40 con 0x560feb71a3a0 2024-04-06T14:17:43.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58718/0 (socket says 172.21.15.90:58718) 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.762+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4155987829 learned_addr learned my addr 172.21.15.90:0/4155987829 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb72ea40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd259c700 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb71a3a0 0x560feb7187e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd259c700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 0x7fafb80a37d0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fafc0011490 tx=0x7fafc0009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4155987829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb71a3a0 msgr2=0x560feb7187e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb71a3a0 0x560feb7187e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4155987829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 msgr2=0x560feb72ea40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb72ea40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4155987829 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560feb4ec3b0 con 0x560feb718d20 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009990 tx=0x7fafc400b5b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c615d943f081f555 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafbcff9700 1 -- 172.21.15.90:0/4155987829 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc401d070 con 0x560feb718d20 2024-04-06T14:17:43.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafbcff9700 1 -- 172.21.15.90:0/4155987829 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fafc401e040 con 0x560feb718d20 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.763+0000 7fafbcff9700 1 -- 172.21.15.90:0/4155987829 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc40047f0 con 0x560feb718d20 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4155987829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 msgr2=0x560feb7191a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 secure :-1 s=READY pgs=785 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009990 tx=0x7fafc400b5b0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4155987829 shutdown_connections 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb72ea40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb71a3a0 0x560feb7187e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4155987829 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb718d20 0x560feb7191a0 unknown :-1 s=CLOSED pgs=785 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4155987829 >> 172.21.15.90:0/4155987829 conn(0x560feb6786a0 msgr2=0x560feb714c30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4155987829 shutdown_connections 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4155987829 wait complete. 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 -- start start 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.764+0000 7fafe09b4b80 1 Processor -- start 2024-04-06T14:17:43.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71f440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71cc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb606d80 con 0x560feb71a3a0 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb606a20 con 0x560feb72c650 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb4ee6a0 con 0x560feb718d20 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafe09b4b80 10 client.0 _ll_register_callbacks cb 0x560feb610470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58730/0 (socket says 172.21.15.90:58730) 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 -- 172.21.15.90:0/1031858389 learned_addr learned my addr 172.21.15.90:0/1031858389 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71cc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 -- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 msgr2=0x560feb71f440 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71f440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 -- 172.21.15.90:0/1031858389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 msgr2=0x560feb71cc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71cc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 -- 172.21.15.90:0/1031858389 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafcc10e2d0 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009350 tx=0x7fafc400b980 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c6e1a197f2ecda6e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafb5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.765+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc401d030 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fafc4004910 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560feb62b380 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fafc4005bc0 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 10 client.17463 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560feb498b60 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560feb62b380 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fafc400fb10 con 0x560feb71a3a0 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 10 client.17463 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 10 client.17463 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:17:43.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 20 client.17463 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb73be30 0x560feb73e220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 4 client.17463 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560feb663700 con 0x560feb73be30 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.766+0000 7fafd259c700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb73be30 0x560feb73e220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.767+0000 7fafb5ffb700 10 client.17463 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.767+0000 7fafd259c700 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb73be30 0x560feb73e220 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f37396630561ed79 server_cookie=b3b1d123ef6ea5eb in_seq=0 out_seq=0 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.767+0000 7fafb5ffb700 1 -- 172.21.15.90:0/1031858389 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560feb663700 con 0x560feb73be30 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.767+0000 7fafb5ffb700 10 client.17463 handle_command_reply: tid=0 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.767+0000 7fafe09b4b80 1 client.17463 shutdown 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 msgr2=0x560feb71c6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 secure :-1 s=READY pgs=786 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009350 tx=0x7fafc400b980 comp rx=0 tx=0).stop 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 shutdown_connections 2024-04-06T14:17:43.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb73be30 0x560feb73e220 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71cc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafb5ffb700 0 client.17463 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71f440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/1031858389 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb71c6c0 unknown :-1 s=CLOSED pgs=786 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 >> 172.21.15.90:0/1031858389 conn(0x560feb6786a0 msgr2=0x560feb714820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 shutdown_connections 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 1 -- 172.21.15.90:0/1031858389 wait complete. 2024-04-06T14:17:43.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.768+0000 7fafe09b4b80 20 client.17463 trim_cache size 0 max 16384 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.783+0000 7fafe09b4b80 1 Processor -- start 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.783+0000 7fafe09b4b80 1 -- start start 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71c660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71ef70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb71ba10 con 0x560feb71a3a0 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb71fc70 con 0x560feb72c650 2024-04-06T14:17:43.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb719100 con 0x560feb718d20 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58736/0 (socket says 172.21.15.90:58736) 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4131629075 learned_addr learned my addr 172.21.15.90:0/4131629075 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71ef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd259c700 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71c660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4131629075 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 msgr2=0x560feb71c660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71c660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4131629075 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 msgr2=0x560feb71ef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71ef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 -- 172.21.15.90:0/4131629075 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560feb630ea0 con 0x560feb71a3a0 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7fafc400b460 tx=0x7fafc402c480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=651ff6821663778d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafbd7fa700 1 -- 172.21.15.90:0/4131629075 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc401d040 con 0x560feb71a3a0 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafbd7fa700 1 -- 172.21.15.90:0/4131629075 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fafc402c5a0 con 0x560feb71a3a0 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.784+0000 7fafbd7fa700 1 -- 172.21.15.90:0/4131629075 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc4004a80 con 0x560feb71a3a0 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4131629075 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 msgr2=0x560feb733490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 secure :-1 s=READY pgs=787 cs=0 l=1 rev1=1 crypto rx=0x7fafc400b460 tx=0x7fafc402c480 comp rx=0 tx=0).stop 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4131629075 shutdown_connections 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb71ef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb71c660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/4131629075 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733490 unknown :-1 s=CLOSED pgs=787 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4131629075 >> 172.21.15.90:0/4131629075 conn(0x560feb6786a0 msgr2=0x560feb71bf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4131629075 shutdown_connections 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.785+0000 7fafe09b4b80 1 -- 172.21.15.90:0/4131629075 wait complete. 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 -- start start 2024-04-06T14:17:43.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 Processor -- start 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb732d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb731210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb731830 con 0x560feb71a3a0 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb7319a0 con 0x560feb72c650 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560feb731b60 con 0x560feb718d20 2024-04-06T14:17:43.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafe09b4b80 10 client.0 _ll_register_callbacks cb 0x560feb610470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb732d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd259c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb732d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51680/0 (socket says 172.21.15.90:51680) 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd259c700 1 -- 172.21.15.90:0/661505610 learned_addr learned my addr 172.21.15.90:0/661505610 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd2d9d700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb731210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd1d9b700 1 -- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 msgr2=0x560feb732d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb732d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.786+0000 7fafd1d9b700 1 -- 172.21.15.90:0/661505610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 msgr2=0x560feb731210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb731210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafd1d9b700 1 -- 172.21.15.90:0/661505610 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560feb498b60 con 0x560feb71a3a0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafd1d9b700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733260 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009990 tx=0x7fafc402ce80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1424d4c8ab8871c5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:43.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fafc401d040 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fafc4004710 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fafc401e040 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fafc4004910 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 10 client.17469 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560feb4f0cd0 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fafc4052020 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fafc400f960 con 0x560feb71a3a0 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 10 client.17469 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 10 client.17469 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 20 client.17469 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb790890 0x560feb792cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 4 client.17469 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.787+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x560feb4b9cf0 con 0x560feb790890 2024-04-06T14:17:43.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafd259c700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb790890 0x560feb792cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafb5ffb700 10 client.17469 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafd259c700 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb790890 0x560feb792cb0 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=f56f3d34de7d9e36 server_cookie=95a062a2d2d85124 in_seq=0 out_seq=0 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafb5ffb700 1 -- 172.21.15.90:0/661505610 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x560feb4b9cf0 con 0x560feb790890 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafb5ffb700 10 client.17469 handle_command_reply: tid=0 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 client.17469 shutdown 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 msgr2=0x560feb733260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733260 secure :-1 s=READY pgs=788 cs=0 l=1 rev1=1 crypto rx=0x7fafc4009990 tx=0x7fafc402ce80 comp rx=0 tx=0).stop 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 shutdown_connections 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x560feb790890 0x560feb792cb0 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560feb72c650 0x560feb731210 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560feb718d20 0x560feb732d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 --2- 172.21.15.90:0/661505610 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560feb71a3a0 0x560feb733260 unknown :-1 s=CLOSED pgs=788 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 >> 172.21.15.90:0/661505610 conn(0x560feb6786a0 msgr2=0x560feb737be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafb5ffb700 0 client.17469 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:17:43.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 shutdown_connections 2024-04-06T14:17:43.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.788+0000 7fafe09b4b80 1 -- 172.21.15.90:0/661505610 wait complete. 2024-04-06T14:17:43.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafe09b4b80 20 client.17469 trim_cache size 0 max 16384 2024-04-06T14:17:43.795 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 msgr2=0x7fafb80a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 0x7fafb80a37d0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fafc0011490 tx=0x7fafc0009260 comp rx=0 tx=0).stop 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 msgr2=0x7fafcc19bd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 secure :-1 s=READY pgs=784 cs=0 l=1 rev1=1 crypto rx=0x7fafc800ef30 tx=0x7fafc800c3a0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 shutdown_connections 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fafb80a1310 0x7fafb80a37d0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fafcc19b910 0x7fafcc19bd90 unknown :-1 s=CLOSED pgs=784 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fafcc100dc0 0x7fafcc197030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 --2- 172.21.15.90:0/3409767546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fafcc105ee0 0x7fafcc197570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:43.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 >> 172.21.15.90:0/3409767546 conn(0x7fafcc0fc660 msgr2=0x7fafcc0fe9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:43.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 shutdown_connections 2024-04-06T14:17:43.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:43.789+0000 7fafd359e700 1 -- 172.21.15.90:0/3409767546 wait complete. 2024-04-06T14:17:43.802 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:17:43.803 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:17:44.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- 172.21.15.90:0/3350705078 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8100820 msgr2=0x7fc5d8104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3350705078 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8100820 0x7fc5d8104840 secure :-1 s=READY pgs=789 cs=0 l=1 rev1=1 crypto rx=0x7fc5c8009990 tx=0x7fc5c8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- 172.21.15.90:0/3350705078 shutdown_connections 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3350705078 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d8104f70 0x7fc5d810caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3350705078 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 0x7fc5d8100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3350705078 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8100820 0x7fc5d8104840 unknown :-1 s=CLOSED pgs=789 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- 172.21.15.90:0/3350705078 >> 172.21.15.90:0/3350705078 conn(0x7fc5d8076d50 msgr2=0x7fc5d8077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- 172.21.15.90:0/3350705078 shutdown_connections 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- 172.21.15.90:0/3350705078 wait complete. 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 Processor -- start 2024-04-06T14:17:44.027 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.027+0000 7fc5de0da700 1 -- start start 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 0x7fc5d8196120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d819aa00 0x7fc5d819ae80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5d819b480 con 0x7fc5d8104f70 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5d819b5f0 con 0x7fc5d819aa00 2024-04-06T14:17:44.029 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc5d819b7b0 con 0x7fc5d80ffe70 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d77fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 0x7fc5d8196120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58764/0 (socket says 172.21.15.90:58764) 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3813733349 learned_addr learned my addr 172.21.15.90:0/3813733349 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d819aa00 0x7fc5d819ae80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3813733349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 msgr2=0x7fc5d8196120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 0x7fc5d8196120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3813733349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d819aa00 msgr2=0x7fc5d819ae80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d819aa00 0x7fc5d819ae80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3813733349 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc5d0009c40 con 0x7fc5d8104f70 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7fc5c8000c00 tx=0x7fc5c800b9a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d4ff9700 1 -- 172.21.15.90:0/3813733349 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c80199b0 con 0x7fc5d8104f70 2024-04-06T14:17:44.030 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d4ff9700 1 -- 172.21.15.90:0/3813733349 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc5c800f460 con 0x7fc5d8104f70 2024-04-06T14:17:44.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc5c80095f0 con 0x7fc5d8104f70 2024-04-06T14:17:44.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5d4ff9700 1 -- 172.21.15.90:0/3813733349 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c8017650 con 0x7fc5d8104f70 2024-04-06T14:17:44.031 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.028+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5d8068090 con 0x7fc5d8104f70 2024-04-06T14:17:44.032 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 Processor -- start 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5d4ff9700 1 -- 172.21.15.90:0/3813733349 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc5c800f5d0 con 0x7fc5d8104f70 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 -- start start 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b108780 0x55c15b108b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b1151d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b060990 con 0x55c15b1090a0 2024-04-06T14:17:44.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b05e350 con 0x55c15b108780 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b059c40 con 0x55c15b112de0 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.033+0000 7fc5d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58770/0 (socket says 172.21.15.90:58770) 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d6ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b1151d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 -- 172.21.15.90:0/1963641618 learned_addr learned my addr 172.21.15.90:0/1963641618 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b108780 0x55c15b108b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d4ff9700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 0x7fc5c40a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 -- 172.21.15.90:0/1963641618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 msgr2=0x55c15b1151d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b1151d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 -- 172.21.15.90:0/1963641618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b108780 msgr2=0x55c15b108b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b108780 0x55c15b108b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 -- 172.21.15.90:0/1963641618 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c15aedb3b0 con 0x55c15b1090a0 2024-04-06T14:17:44.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d4ff9700 1 -- 172.21.15.90:0/3813733349 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc5c80ce3d0 con 0x7fc5d8104f70 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 0x7fc5c40a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0010e20 tx=0x7fc5c0009260 comp rx=0 tx=0).ready entity=mon.0 client_cookie=efd3d9f33bc82161 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5cdffb700 1 -- 172.21.15.90:0/1963641618 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c0015050 con 0x55c15b1090a0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5cdffb700 1 -- 172.21.15.90:0/1963641618 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc5c0016040 con 0x55c15b1090a0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.034+0000 7fc5cdffb700 1 -- 172.21.15.90:0/1963641618 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c001a400 con 0x55c15b1090a0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/1963641618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 msgr2=0x55c15b1128a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 0x7fc5c40a3750 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0009fd0 tx=0x7fc5d0004ad0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 secure :-1 s=READY pgs=791 cs=0 l=1 rev1=1 crypto rx=0x7fc5c0010e20 tx=0x7fc5c0009260 comp rx=0 tx=0).stop 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/1963641618 shutdown_connections 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b1151d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b108780 0x55c15b108b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/1963641618 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b1128a0 unknown :-1 s=CLOSED pgs=791 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/1963641618 >> 172.21.15.90:0/1963641618 conn(0x55c15b0676a0 msgr2=0x55c15b103a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/1963641618 shutdown_connections 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/1963641618 wait complete. 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.035+0000 7fc5ea28eb80 1 -- start start 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 Processor -- start 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b108780 0x55c15b112860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b1090a0 0x55c15b10fb50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15aff5d80 con 0x55c15b108780 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15aff5a20 con 0x55c15b1090a0 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15aedd6a0 con 0x55c15b112de0 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5ea28eb80 10 client.0 _ll_register_callbacks cb 0x55c15afff470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d77fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b108780 0x55c15b112860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51710/0 (socket says 172.21.15.90:51710) 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2384536916 learned_addr learned my addr 172.21.15.90:0/2384536916 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b1090a0 0x55c15b10fb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b1090a0 msgr2=0x55c15b10fb50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b1090a0 0x55c15b10fb50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2384536916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b108780 msgr2=0x55c15b112860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b108780 0x55c15b112860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.036+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2384536916 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5c0010b00 con 0x55c15b112de0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.037+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0017e10 tx=0x7fc5d000b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=15dbcd06cb743119 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.037+0000 7fc5b6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.037+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5d0027070 con 0x55c15b112de0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.037+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc5d0017e40 con 0x55c15b112de0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.037+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c15b01a380 con 0x55c15b112de0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc5d000e040 con 0x55c15b112de0 2024-04-06T14:17:44.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 10 client.26992 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:44.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c15ae87b60 con 0x55c15b112de0 2024-04-06T14:17:44.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c15b01a380 con 0x55c15b112de0 2024-04-06T14:17:44.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc5d002a020 con 0x55c15b112de0 2024-04-06T14:17:44.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 10 client.26992 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 10 client.26992 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 20 client.26992 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.038+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b122af0 0x55c15b124f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.039+0000 7fc5ea28eb80 4 client.26992 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.039+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c15b052700 con 0x55c15b122af0 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.039+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b122af0 0x55c15b124f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.039+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b122af0 0x55c15b124f10 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f8fcb5dbca1ff721 server_cookie=767d9676844de38a in_seq=0 out_seq=0 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.039+0000 7fc5b6ffd700 10 client.26992 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.040+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2384536916 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c15b052700 con 0x55c15b122af0 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.040+0000 7fc5b6ffd700 10 client.26992 handle_command_reply: tid=0 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.040+0000 7fc5ea28eb80 1 client.26992 shutdown 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 msgr2=0x55c15b110240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0017e10 tx=0x7fc5d000b040 comp rx=0 tx=0).stop 2024-04-06T14:17:44.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 shutdown_connections 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b122af0 0x55c15b124f10 crc :-1 s=READY pgs=211 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b112de0 0x55c15b110240 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b108780 0x55c15b112860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2384536916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b1090a0 0x55c15b10fb50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 >> 172.21.15.90:0/2384536916 conn(0x55c15b0676a0 msgr2=0x55c15b102c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5b6ffd700 0 client.26992 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 shutdown_connections 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.043+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2384536916 wait complete. 2024-04-06T14:17:44.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.044+0000 7fc5ea28eb80 20 client.26992 trim_cache size 0 max 16384 2024-04-06T14:17:44.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.062+0000 7fc5ea28eb80 1 Processor -- start 2024-04-06T14:17:44.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.062+0000 7fc5ea28eb80 1 -- start start 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.062+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b108780 0x55c15b110150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b10f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b1047f0 con 0x55c15b1090a0 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b119fa0 con 0x55c15b112de0 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b110840 con 0x55c15b108780 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58796/0 (socket says 172.21.15.90:58796) 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3506555222 learned_addr learned my addr 172.21.15.90:0/3506555222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3506555222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b108780 msgr2=0x55c15b110150 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b108780 0x55c15b110150 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3506555222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 msgr2=0x55c15b10f3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b10f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.063+0000 7fc5d6ffd700 1 -- 172.21.15.90:0/3506555222 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c15b01fea0 con 0x55c15b1090a0 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.064+0000 7fc5d6ffd700 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fc5c800fcc0 tx=0x7fc5c80d1ee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e0ebd1093025c073 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.064+0000 7fc5ce7fc700 1 -- 172.21.15.90:0/3506555222 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c8097890 con 0x55c15b1090a0 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.064+0000 7fc5ce7fc700 1 -- 172.21.15.90:0/3506555222 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc5c809b320 con 0x55c15b1090a0 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.064+0000 7fc5ce7fc700 1 -- 172.21.15.90:0/3506555222 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5c809f160 con 0x55c15b1090a0 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.065+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/3506555222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 msgr2=0x55c15b119a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.065+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 secure :-1 s=READY pgs=792 cs=0 l=1 rev1=1 crypto rx=0x7fc5c800fcc0 tx=0x7fc5c80d1ee0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.075+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/3506555222 shutdown_connections 2024-04-06T14:17:44.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.075+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b10f3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.080 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.075+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b108780 0x55c15b110150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.075+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/3506555222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119a60 unknown :-1 s=CLOSED pgs=792 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.075+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/3506555222 >> 172.21.15.90:0/3506555222 conn(0x55c15b0676a0 msgr2=0x55c15b129870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.077+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/3506555222 shutdown_connections 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.077+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/3506555222 wait complete. 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.077+0000 7fc5ea28eb80 1 -- start start 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.077+0000 7fc5ea28eb80 1 Processor -- start 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b1197b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b118400 con 0x55c15b1090a0 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b118540 con 0x55c15b112de0 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c15b118700 con 0x55c15b117a40 2024-04-06T14:17:44.081 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5ea28eb80 10 client.0 _ll_register_callbacks cb 0x55c15afff470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5d7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5d7fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51714/0 (socket says 172.21.15.90:51714) 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.078+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2725465326 learned_addr learned my addr 172.21.15.90:0/2725465326 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 msgr2=0x55c15b1197b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b1197b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2725465326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 msgr2=0x55c15b119270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 -- 172.21.15.90:0/2725465326 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc5c8097040 con 0x55c15b117a40 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5d7fff700 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0007f30 tx=0x7fc5d0007f60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=25d6fba272e98e96 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:44.082 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.079+0000 7fc5b6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:44.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.089+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc5d0027070 con 0x55c15b117a40 2024-04-06T14:17:44.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.089+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc5d0004d50 con 0x55c15b117a40 2024-04-06T14:17:44.089 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.089+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc5c0016040 con 0x55c15b117a40 2024-04-06T14:17:44.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.168+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc5d0016a10 con 0x55c15b117a40 2024-04-06T14:17:44.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.168+0000 7fc5ea28eb80 10 client.26995 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:44.168 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.168+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c15aedfcd0 con 0x55c15b117a40 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc5c0016040 con 0x55c15b117a40 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc5d003d070 con 0x55c15b117a40 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 10 client.26995 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 10 client.26995 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 20 client.26995 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:44.172 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b16dce0 0x55c15b170100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:44.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 4 client.26995 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:17:44.173 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.173+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c15aea8cf0 con 0x55c15b16dce0 2024-04-06T14:17:44.177 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.178+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b16dce0 0x55c15b170100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:44.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.190+0000 7fc5d77fe700 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b16dce0 0x55c15b170100 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e8bf000e84413091 server_cookie=db9510f440c775e7 in_seq=0 out_seq=0 2024-04-06T14:17:44.189 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.190+0000 7fc5b6ffd700 10 client.26995 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:44.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.190+0000 7fc5b6ffd700 1 -- 172.21.15.90:0/2725465326 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c15aea8cf0 con 0x55c15b16dce0 2024-04-06T14:17:44.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.190+0000 7fc5b6ffd700 10 client.26995 handle_command_reply: tid=0 2024-04-06T14:17:44.190 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.190+0000 7fc5ea28eb80 1 client.26995 shutdown 2024-04-06T14:17:44.191 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.192+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 msgr2=0x55c15b117ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.191 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.192+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0007f30 tx=0x7fc5d0007f60 comp rx=0 tx=0).stop 2024-04-06T14:17:44.193 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 shutdown_connections 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55c15b16dce0 0x55c15b170100 crc :-1 s=READY pgs=212 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c15b117a40 0x55c15b117ec0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c15b1090a0 0x55c15b119270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 --2- 172.21.15.90:0/2725465326 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c15b112de0 0x55c15b1197b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 >> 172.21.15.90:0/2725465326 conn(0x55c15b0676a0 msgr2=0x55c15b10f3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.194 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.194+0000 7fc5b6ffd700 0 client.26995 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:44.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.196+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 shutdown_connections 2024-04-06T14:17:44.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.196+0000 7fc5ea28eb80 1 -- 172.21.15.90:0/2725465326 wait complete. 2024-04-06T14:17:44.196 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.197+0000 7fc5ea28eb80 20 client.26995 trim_cache size 0 max 16384 2024-04-06T14:17:44.197 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:17:44.197 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.197+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 msgr2=0x7fc5c40a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.197+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 0x7fc5c40a3750 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fc5d0009fd0 tx=0x7fc5d0004ad0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.197+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 msgr2=0x7fc5d8196660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.197+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 secure :-1 s=READY pgs=790 cs=0 l=1 rev1=1 crypto rx=0x7fc5c8000c00 tx=0x7fc5c800b9a0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 shutdown_connections 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc5c40a1290 0x7fc5c40a3750 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc5d819aa00 0x7fc5d819ae80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc5d80ffe70 0x7fc5d8196120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 --2- 172.21.15.90:0/3813733349 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc5d8104f70 0x7fc5d8196660 unknown :-1 s=CLOSED pgs=790 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 >> 172.21.15.90:0/3813733349 conn(0x7fc5d8076d50 msgr2=0x7fc5d80fde40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 shutdown_connections 2024-04-06T14:17:44.198 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:44.198+0000 7fc5de0da700 1 -- 172.21.15.90:0/3813733349 wait complete. 2024-04-06T14:17:44.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:44 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2124893855' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:44.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:44 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2124893855' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:44.677 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:44 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2124893855' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:17:45.215 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:17:45.215 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:17:45.215 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:17:45.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- 172.21.15.90:0/1380679861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c11a590 msgr2=0x7f1d9c11c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 --2- 172.21.15.90:0/1380679861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c11a590 0x7f1d9c11c980 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f1d94009f90 tx=0x7f1d9400b330 comp rx=0 tx=0).stop 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- 172.21.15.90:0/1380679861 shutdown_connections 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 --2- 172.21.15.90:0/1380679861 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c11a590 0x7f1d9c11c980 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 --2- 172.21.15.90:0/1380679861 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f4b0 0x7f1d9c107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 --2- 172.21.15.90:0/1380679861 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c10f920 0x7f1d9c1082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- 172.21.15.90:0/1380679861 >> 172.21.15.90:0/1380679861 conn(0x7f1d9c06be00 msgr2=0x7f1d9c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- 172.21.15.90:0/1380679861 shutdown_connections 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- 172.21.15.90:0/1380679861 wait complete. 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 Processor -- start 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.414+0000 7f1da2f16700 1 -- start start 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c10f4b0 0x7f1d9c1a6110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f920 0x7f1d9c1a6650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d9c1ab030 con 0x7f1d9c10f920 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d9c1ab1a0 con 0x7f1d9c11a590 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d9c1ab360 con 0x7f1d9c10f4b0 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34350/0 (socket says 172.21.15.90:34350) 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 -- 172.21.15.90:0/947195084 learned_addr learned my addr 172.21.15.90:0/947195084 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c10f4b0 0x7f1d9c1a6110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f920 0x7f1d9c1a6650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 -- 172.21.15.90:0/947195084 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c10f4b0 msgr2=0x7f1d9c1a6110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c10f4b0 0x7f1d9c1a6110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 -- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f920 msgr2=0x7f1d9c1a6650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f920 0x7f1d9c1a6650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 -- 172.21.15.90:0/947195084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d94009c40 con 0x7f1d9c11a590 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da14b3700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f1d94009f60 tx=0x7f1d940094d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1d99ffb700 1 -- 172.21.15.90:0/947195084 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d9400e040 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d9c1ab5e0 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1d99ffb700 1 -- 172.21.15.90:0/947195084 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d94007e40 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.415+0000 7f1d99ffb700 1 -- 172.21.15.90:0/947195084 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d94005040 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.416+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d9c1b2e80 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.419+0000 7f1d99ffb700 1 -- 172.21.15.90:0/947195084 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1d9400bbf0 con 0x7f1d9c11a590 2024-04-06T14:17:45.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.419+0000 7f1d99ffb700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 0x7f1d8c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.420+0000 7f1d99ffb700 1 -- 172.21.15.90:0/947195084 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d940cdf60 con 0x7f1d9c11a590 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.420+0000 7f1daf0cab80 1 Processor -- start 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.420+0000 7f1daf0cab80 1 -- start start 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.420+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c3abe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.420+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 0x564b97c47250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97b92a10 con 0x564b97c3b120 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97b903d0 con 0x564b97c44e60 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97b8bcc0 con 0x564b97c3a800 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58826/0 (socket says 172.21.15.90:58826) 2024-04-06T14:17:45.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2167366235 learned_addr learned my addr 172.21.15.90:0/2167366235 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1da14b3700 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 0x564b97c47250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2167366235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 msgr2=0x564b97c3abe0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c3abe0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2167366235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 msgr2=0x564b97c47250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 0x564b97c47250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2167366235 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564b97a0d430 con 0x564b97c3b120 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1da14b3700 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 0x564b97c47250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f1d88009cb0 tx=0x7f1d88011440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6339768f96496201 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d86ffd700 1 -- 172.21.15.90:0/2167366235 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d88011d20 con 0x564b97c3b120 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d86ffd700 1 -- 172.21.15.90:0/2167366235 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d88011e80 con 0x564b97c3b120 2024-04-06T14:17:45.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.421+0000 7f1d86ffd700 1 -- 172.21.15.90:0/2167366235 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d88005280 con 0x564b97c3b120 2024-04-06T14:17:45.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 0x7f1d8c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2167366235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 msgr2=0x564b97c44920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 secure :-1 s=READY pgs=793 cs=0 l=1 rev1=1 crypto rx=0x7f1d88009cb0 tx=0x7f1d88011440 comp rx=0 tx=0).stop 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2167366235 shutdown_connections 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c44e60 0x564b97c47250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c3abe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2167366235 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c3b120 0x564b97c44920 unknown :-1 s=CLOSED pgs=793 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2167366235 >> 172.21.15.90:0/2167366235 conn(0x564b97b99720 msgr2=0x564b97c35b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2167366235 shutdown_connections 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2167366235 wait complete. 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.422+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 0x7f1d8c0a37d0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1d90005fd0 tx=0x7f1d90009310 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.423+0000 7f1daf0cab80 1 -- start start 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.423+0000 7f1daf0cab80 1 Processor -- start 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c42150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c42690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97b27e00 con 0x564b97c44e60 2024-04-06T14:17:45.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97b27aa0 con 0x564b97c3b120 2024-04-06T14:17:45.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97a0f720 con 0x564b97c3a800 2024-04-06T14:17:45.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1daf0cab80 10 client.0 _ll_register_callbacks cb 0x564b97b314f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:45.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1da0cb2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1da0cb2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:51732/0 (socket says 172.21.15.90:51732) 2024-04-06T14:17:45.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1da0cb2700 1 -- 172.21.15.90:0/4247145564 learned_addr learned my addr 172.21.15.90:0/4247145564 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.424+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c42150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 -- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 msgr2=0x564b97c42150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c42150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 -- 172.21.15.90:0/4247145564 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 msgr2=0x564b97c42690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c42690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 -- 172.21.15.90:0/4247145564 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d9c1b2e80 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f1d9000e6e0 tx=0x7f1d900047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7780b0c865ab7648 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1d7bfff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d90004a30 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564b97b84780 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d90004b90 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d90004d90 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1daf0cab80 10 client.27004 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.425+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f1d880098e0 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564b97b84780 con 0x564b97c3a800 2024-04-06T14:17:45.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1d90026830 con 0x564b97c3a800 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 10 client.27004 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 10 client.27004 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 20 client.27004 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97c54ac0 0x564b97c56ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 4 client.27004 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564b979dad70 con 0x564b97c54ac0 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.426+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97c54ac0 0x564b97c56ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.427+0000 7f1d7bfff700 10 client.27004 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.427+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97c54ac0 0x564b97c56ee0 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9824ef0ee1042ef1 server_cookie=a10869d7da9a9129 in_seq=0 out_seq=0 2024-04-06T14:17:45.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1d7bfff700 1 -- 172.21.15.90:0/4247145564 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564b979dad70 con 0x564b97c54ac0 2024-04-06T14:17:45.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1d7bfff700 10 client.27004 handle_command_reply: tid=0 2024-04-06T14:17:45.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 client.27004 shutdown 2024-04-06T14:17:45.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 msgr2=0x564b97c41c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f1d9000e6e0 tx=0x7f1d900047c0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 shutdown_connections 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c42690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97c54ac0 0x564b97c56ee0 crc :-1 s=READY pgs=213 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c41c10 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/4247145564 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c42150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 >> 172.21.15.90:0/4247145564 conn(0x564b97b99720 msgr2=0x564b97c34ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1d7bfff700 0 client.27004 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 shutdown_connections 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 1 -- 172.21.15.90:0/4247145564 wait complete. 2024-04-06T14:17:45.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.428+0000 7f1daf0cab80 20 client.27004 trim_cache size 0 max 16384 2024-04-06T14:17:45.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.448+0000 7f1daf0cab80 1 Processor -- start 2024-04-06T14:17:45.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.448+0000 7f1daf0cab80 1 -- start start 2024-04-06T14:17:45.448 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c424b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c3ee90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97c2d440 con 0x564b97c44e60 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97c40160 con 0x564b97c3b120 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97c43190 con 0x564b97c3a800 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34370/0 (socket says 172.21.15.90:34370) 2024-04-06T14:17:45.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2160774129 learned_addr learned my addr 172.21.15.90:0/2160774129 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c424b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.449+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2160774129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 msgr2=0x564b97c424b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c424b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2160774129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 msgr2=0x564b97c3ee90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c3ee90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d9bfff700 1 -- 172.21.15.90:0/2160774129 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564b979b9be0 con 0x564b97c3b120 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c424b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:17:45.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f1d88009bd0 tx=0x7f1d88004c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d22fc181b1d69c05 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d877fe700 1 -- 172.21.15.90:0/2160774129 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d88004890 con 0x564b97c3b120 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d877fe700 1 -- 172.21.15.90:0/2160774129 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d880049f0 con 0x564b97c3b120 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.450+0000 7f1d877fe700 1 -- 172.21.15.90:0/2160774129 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d88003e80 con 0x564b97c3b120 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2160774129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 msgr2=0x564b97c40bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7f1d88009bd0 tx=0x7f1d88004c00 comp rx=0 tx=0).stop 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2160774129 shutdown_connections 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97c3ee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c424b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/2160774129 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c40bf0 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2160774129 >> 172.21.15.90:0/2160774129 conn(0x564b97b99720 msgr2=0x564b97c404e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2160774129 shutdown_connections 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.451+0000 7f1daf0cab80 1 -- 172.21.15.90:0/2160774129 wait complete. 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 -- start start 2024-04-06T14:17:45.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 Processor -- start 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c4aac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97ca28f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97ca2e30 con 0x564b97c44e60 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97ca2fa0 con 0x564b97c3b120 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564b97ca3160 con 0x564b97c3a800 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1daf0cab80 10 client.0 _ll_register_callbacks cb 0x564b97b314f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:34382/0 (socket says 172.21.15.90:34382) 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 -- 172.21.15.90:0/811070366 learned_addr learned my addr 172.21.15.90:0/811070366 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c4aac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1da14b3700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97ca28f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 -- 172.21.15.90:0/811070366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 msgr2=0x564b97c4aac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c4aac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 -- 172.21.15.90:0/811070366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 msgr2=0x564b97ca28f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97ca28f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.452+0000 7f1d9bfff700 1 -- 172.21.15.90:0/811070366 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d8801d040 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d9bfff700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f1d88011790 tx=0x7f1d88003680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=54306dd09e32ccb9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d88003cc0 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564b97b4c400 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d88003e20 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d88003920 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 10 client.27236 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564b97b51f20 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564b97b4c400 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1d8802b070 con 0x564b97c3b120 2024-04-06T14:17:45.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 10 client.27236 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 10 client.27236 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 20 client.27236 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97ca43e0 0x564b97cbd930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 4 client.27236 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.453+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x564b97a0fd60 con 0x564b97ca43e0 2024-04-06T14:17:45.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.454+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97ca43e0 0x564b97cbd930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.454+0000 7f1d7bfff700 10 client.27236 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.454+0000 7f1da0cb2700 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97ca43e0 0x564b97cbd930 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e6038987cc69d5bd server_cookie=1cd717efe76d1a9c in_seq=0 out_seq=0 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1d7bfff700 1 -- 172.21.15.90:0/811070366 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x564b97a0fd60 con 0x564b97ca43e0 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1d7bfff700 10 client.27236 handle_command_reply: tid=0 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 client.27236 shutdown 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 msgr2=0x564b97c4b000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f1d88011790 tx=0x7f1d88003680 comp rx=0 tx=0).stop 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 shutdown_connections 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564b97c44e60 0x564b97ca28f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564b97ca43e0 0x564b97cbd930 crc :-1 s=READY pgs=214 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564b97c3a800 0x564b97c4aac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 --2- 172.21.15.90:0/811070366 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564b97c3b120 0x564b97c4b000 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 >> 172.21.15.90:0/811070366 conn(0x564b97b99720 msgr2=0x564b97c3a080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.455+0000 7f1d7bfff700 0 client.27236 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 shutdown_connections 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1daf0cab80 1 -- 172.21.15.90:0/811070366 wait complete. 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1daf0cab80 20 client.27236 trim_cache size 0 max 16384 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "f1207eab-ce7d-4b96-806c-58df855fad4d", 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:17:45.460 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 msgr2=0x7f1d8c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 0x7f1d8c0a37d0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f1d90005fd0 tx=0x7f1d90009310 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 msgr2=0x7f1d9c1aa9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7f1d94009f60 tx=0x7f1d940094d0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 shutdown_connections 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d8c0a1310 0x7f1d8c0a37d0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d9c11a590 0x7f1d9c1aa9f0 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d9c10f4b0 0x7f1d9c1a6110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 --2- 172.21.15.90:0/947195084 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d9c10f920 0x7f1d9c1a6650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.456+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 >> 172.21.15.90:0/947195084 conn(0x7f1d9c06be00 msgr2=0x7f1d9c118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.457+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 shutdown_connections 2024-04-06T14:17:45.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.457+0000 7f1da2f16700 1 -- 172.21.15.90:0/947195084 wait complete. 2024-04-06T14:17:45.490 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:17:45.491 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:17:45.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:45 smithi105 ceph-mon[46969]: pgmap v385: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.07k op/s 2024-04-06T14:17:45.507 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:45 smithi090 ceph-mon[43420]: pgmap v385: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.07k op/s 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 -- 172.21.15.90:0/1888874730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa8480fff90 msgr2=0x7fa848103fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 --2- 172.21.15.90:0/1888874730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa8480fff90 0x7fa848103fd0 secure :-1 s=READY pgs=794 cs=0 l=1 rev1=1 crypto rx=0x7fa84400aaa0 tx=0x7fa84400adb0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 -- 172.21.15.90:0/1888874730 shutdown_connections 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 --2- 172.21.15.90:0/1888874730 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa848104700 0x7fa84810c280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 --2- 172.21.15.90:0/1888874730 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa8480ff5e0 0x7fa8480ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 --2- 172.21.15.90:0/1888874730 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa8480fff90 0x7fa848103fd0 unknown :-1 s=CLOSED pgs=794 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.735+0000 7fa850674700 1 -- 172.21.15.90:0/1888874730 >> 172.21.15.90:0/1888874730 conn(0x7fa8480761b0 msgr2=0x7fa8480fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.739+0000 7fa850674700 1 -- 172.21.15.90:0/1888874730 shutdown_connections 2024-04-06T14:17:45.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.739+0000 7fa850674700 1 -- 172.21.15.90:0/1888874730 wait complete. 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 Processor -- start 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 -- start start 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa848197a00 0x7fa84819bd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa84819c290 0x7fa84819c740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8481980f0 con 0x7fa848104700 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8481a4000 con 0x7fa848197a00 2024-04-06T14:17:45.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.741+0000 7fa850674700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8481a4140 con 0x7fa84819c290 2024-04-06T14:17:45.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58870/0 (socket says 172.21.15.90:58870) 2024-04-06T14:17:45.743 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 -- 172.21.15.90:0/3036701791 learned_addr learned my addr 172.21.15.90:0/3036701791 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 -- 172.21.15.90:0/3036701791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa84819c290 msgr2=0x7fa84819c740 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:17:45.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa84819c290 0x7fa84819c740 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 -- 172.21.15.90:0/3036701791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa848197a00 msgr2=0x7fa84819bd50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa848197a00 0x7fa84819bd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.743+0000 7fa84e410700 1 -- 172.21.15.90:0/3036701791 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa84400a700 con 0x7fa848104700 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.744+0000 7fa84e410700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7fa83800b480 tx=0x7fa83800b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.744+0000 7fa83f7fe700 1 -- 172.21.15.90:0/3036701791 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa838011630 con 0x7fa848104700 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.744+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa848108380 con 0x7fa848104700 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.744+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8481088e0 con 0x7fa848104700 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.745+0000 7fa83f7fe700 1 -- 172.21.15.90:0/3036701791 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa838011c50 con 0x7fa848104700 2024-04-06T14:17:45.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.745+0000 7fa83f7fe700 1 -- 172.21.15.90:0/3036701791 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa838019c60 con 0x7fa848104700 2024-04-06T14:17:45.748 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.749+0000 7fa85c828b80 1 Processor -- start 2024-04-06T14:17:45.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 -- start start 2024-04-06T14:17:45.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa83f7fe700 1 -- 172.21.15.90:0/3036701791 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa838010380 con 0x7fa848104700 2024-04-06T14:17:45.750 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7d780 0x555babc7db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7e0a0 0x555babc900b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babbd5990 con 0x555babc905f0 2024-04-06T14:17:45.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babbd3350 con 0x555babc7d780 2024-04-06T14:17:45.751 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa85c828b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babbcec40 con 0x555babc7e0a0 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa84dc0f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa84dc0f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58886/0 (socket says 172.21.15.90:58886) 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa84dc0f700 1 -- 172.21.15.90:0/3541733693 learned_addr learned my addr 172.21.15.90:0/3541733693 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.751+0000 7fa84ec11700 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7d780 0x555babc7db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 -- 172.21.15.90:0/3541733693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7e0a0 msgr2=0x555babc900b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7e0a0 0x555babc900b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 -- 172.21.15.90:0/3541733693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7d780 msgr2=0x555babc7db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7d780 0x555babc7db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 -- 172.21.15.90:0/3541733693 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555baba503b0 con 0x555babc905f0 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa83f7fe700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 0x7fa8340a3850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa84dc0f700 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7fa84400a3f0 tx=0x7fa844005ef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5c2dcb58944cb7e1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa83f7fe700 1 -- 172.21.15.90:0/3036701791 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa8380cc020 con 0x7fa848104700 2024-04-06T14:17:45.752 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa83cff9700 1 -- 172.21.15.90:0/3541733693 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa844003880 con 0x555babc905f0 2024-04-06T14:17:45.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa83cff9700 1 -- 172.21.15.90:0/3541733693 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa84400cc40 con 0x555babc905f0 2024-04-06T14:17:45.753 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.752+0000 7fa83cff9700 1 -- 172.21.15.90:0/3541733693 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa84400ea60 con 0x555babc905f0 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 -- 172.21.15.90:0/3541733693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 msgr2=0x555babc929e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 secure :-1 s=READY pgs=796 cs=0 l=1 rev1=1 crypto rx=0x7fa84400a3f0 tx=0x7fa844005ef0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa84ec11700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 0x7fa8340a3850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 -- 172.21.15.90:0/3541733693 shutdown_connections 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc929e0 unknown :-1 s=CLOSED pgs=796 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7d780 0x555babc7db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 --2- 172.21.15.90:0/3541733693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7e0a0 0x555babc900b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 -- 172.21.15.90:0/3541733693 >> 172.21.15.90:0/3541733693 conn(0x555babbdc6a0 msgr2=0x555babc78a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.753+0000 7fa85c828b80 1 -- 172.21.15.90:0/3541733693 shutdown_connections 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 -- 172.21.15.90:0/3541733693 wait complete. 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 -- start start 2024-04-06T14:17:45.755 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 Processor -- start 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa84ec11700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 0x7fa8340a3850 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa840009c10 tx=0x7fa840009040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc90020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babb6ad80 con 0x555babc905f0 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babb6aa20 con 0x555babc7e0a0 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555baba526a0 con 0x555babc7d780 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.754+0000 7fa85c828b80 10 client.0 _ll_register_callbacks cb 0x555babb74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.755+0000 7fa84ec11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.755+0000 7fa84ec11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58894/0 (socket says 172.21.15.90:58894) 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.755+0000 7fa84ec11700 1 -- 172.21.15.90:0/4164040894 learned_addr learned my addr 172.21.15.90:0/4164040894 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.756 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.755+0000 7fa84e410700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc90020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 -- 172.21.15.90:0/4164040894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 msgr2=0x555babc90020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc90020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.757 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 -- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 msgr2=0x555babc8d3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 -- 172.21.15.90:0/4164040894 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa84810caf0 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.756+0000 7fa84ec11700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7fa840007d00 tx=0x7fa840004ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d41bdcce920ee940 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.757+0000 7fa831ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.757+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa84001e070 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.757+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa840004750 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.757+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555babb8f380 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.757+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa840003ab0 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 10 client.17508 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555bab9fcb60 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555babb8f380 con 0x555babc905f0 2024-04-06T14:17:45.758 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa840023530 con 0x555babc905f0 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 10 client.17508 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 10 client.17508 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 20 client.17508 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babca0280 0x555babca26a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 4 client.17508 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.758+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555babbc7700 con 0x555babca0280 2024-04-06T14:17:45.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.759+0000 7fa84e410700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babca0280 0x555babca26a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.759+0000 7fa84e410700 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babca0280 0x555babca26a0 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6eb7fb4e27058bc server_cookie=d91f15402adf9559 in_seq=0 out_seq=0 2024-04-06T14:17:45.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.759+0000 7fa831ffb700 10 client.17508 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.762+0000 7fa831ffb700 1 -- 172.21.15.90:0/4164040894 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555babbc7700 con 0x555babca0280 2024-04-06T14:17:45.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.762+0000 7fa831ffb700 10 client.17508 handle_command_reply: tid=0 2024-04-06T14:17:45.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.762+0000 7fa85c828b80 1 client.17508 shutdown 2024-04-06T14:17:45.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.762+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 msgr2=0x555babc8daf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.762+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 secure :-1 s=READY pgs=797 cs=0 l=1 rev1=1 crypto rx=0x7fa840007d00 tx=0x7fa840004ad0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 shutdown_connections 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc8daf0 unknown :-1 s=CLOSED pgs=797 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babca0280 0x555babca26a0 crc :-1 s=READY pgs=215 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc90020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4164040894 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 >> 172.21.15.90:0/4164040894 conn(0x555babbdc6a0 msgr2=0x555babc77c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa831ffb700 0 client.17508 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 shutdown_connections 2024-04-06T14:17:45.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.766+0000 7fa85c828b80 1 -- 172.21.15.90:0/4164040894 wait complete. 2024-04-06T14:17:45.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.767+0000 7fa85c828b80 20 client.17508 trim_cache size 0 max 16384 2024-04-06T14:17:45.771 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:45 smithi134 ceph-mon[47549]: pgmap v385: 97 pgs: 97 active+clean; 6.3 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.07k op/s 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.789+0000 7fa85c828b80 1 Processor -- start 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 -- start start 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc905f0 0x555babc972a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc7ea10 con 0x555babc7d780 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc8e0f0 con 0x555babc7e0a0 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa85c828b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc977e0 con 0x555babc905f0 2024-04-06T14:17:45.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa84e410700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa84e410700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58900/0 (socket says 172.21.15.90:58900) 2024-04-06T14:17:45.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.790+0000 7fa84e410700 1 -- 172.21.15.90:0/4229206325 learned_addr learned my addr 172.21.15.90:0/4229206325 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 -- 172.21.15.90:0/4229206325 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc905f0 msgr2=0x555babc972a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc905f0 0x555babc972a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 -- 172.21.15.90:0/4229206325 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 msgr2=0x555babc8d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84dc0f700 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 -- 172.21.15.90:0/4229206325 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555babb94ea0 con 0x555babc7d780 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.791+0000 7fa84e410700 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7fa838098ec0 tx=0x7fa838099080 comp rx=0 tx=0).ready entity=mon.0 client_cookie=959ad7f1fb45a6f4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.792+0000 7fa83d7fa700 1 -- 172.21.15.90:0/4229206325 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8380a1640 con 0x555babc7d780 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.792+0000 7fa83d7fa700 1 -- 172.21.15.90:0/4229206325 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa8380b5070 con 0x555babc7d780 2024-04-06T14:17:45.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.792+0000 7fa83d7fa700 1 -- 172.21.15.90:0/4229206325 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8380975f0 con 0x555babc7d780 2024-04-06T14:17:45.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- 172.21.15.90:0/4229206325 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 msgr2=0x555babc929e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 secure :-1 s=READY pgs=798 cs=0 l=1 rev1=1 crypto rx=0x7fa838098ec0 tx=0x7fa838099080 comp rx=0 tx=0).stop 2024-04-06T14:17:45.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- 172.21.15.90:0/4229206325 shutdown_connections 2024-04-06T14:17:45.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc905f0 0x555babc972a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc7d780 0x555babc929e0 unknown :-1 s=CLOSED pgs=798 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 --2- 172.21.15.90:0/4229206325 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc8d350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- 172.21.15.90:0/4229206325 >> 172.21.15.90:0/4229206325 conn(0x555babbdc6a0 msgr2=0x555babc8cc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- 172.21.15.90:0/4229206325 shutdown_connections 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- 172.21.15.90:0/4229206325 wait complete. 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 -- start start 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.793+0000 7fa85c828b80 1 Processor -- start 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc96ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc952c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc95d40 con 0x555babc905f0 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc95eb0 con 0x555babc7e0a0 2024-04-06T14:17:45.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555babc96070 con 0x555babc7d780 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa85c828b80 10 client.0 _ll_register_callbacks cb 0x555babb74470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa84ec11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa84ec11700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:58908/0 (socket says 172.21.15.90:58908) 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa84ec11700 1 -- 172.21.15.90:0/1066907621 learned_addr learned my addr 172.21.15.90:0/1066907621 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa84e410700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc96ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.794+0000 7fa84dc0f700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc952c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 -- 172.21.15.90:0/1066907621 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 msgr2=0x555babc96ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc96ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 -- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 msgr2=0x555babc952c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc952c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 -- 172.21.15.90:0/1066907621 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8481088e0 con 0x555babc905f0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa84ec11700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7fa8400061e0 tx=0x7fa840007860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2d371462c15afda6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa831ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa840032070 con 0x555babc905f0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.795+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(96..96 src has 1..96) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa8400048b0 con 0x555babc905f0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa84400cc40 con 0x555babc905f0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa84001e070 con 0x555babc905f0 2024-04-06T14:17:45.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa85c828b80 10 client.17514 fetch_fsmap learned FSMap version 24 2024-04-06T14:17:45.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555baba54cd0 con 0x555babc905f0 2024-04-06T14:17:45.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa84400cc40 con 0x555babc905f0 2024-04-06T14:17:45.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.796+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa840035030 con 0x555babc905f0 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 10 client.17514 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 10 client.17514 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 20 client.17514 populate_metadata read hostname 'smithi090' 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babcf5540 0x555babcf7960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 4 client.17514 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x555baba1dcf0 con 0x555babcf5540 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.797+0000 7fa84e410700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babcf5540 0x555babcf7960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.798+0000 7fa84e410700 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babcf5540 0x555babcf7960 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bae3507357e328c7 server_cookie=d661197a281e794d in_seq=0 out_seq=0 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.798+0000 7fa831ffb700 10 client.17514 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.798+0000 7fa831ffb700 1 -- 172.21.15.90:0/1066907621 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x555baba1dcf0 con 0x555babcf5540 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.798+0000 7fa831ffb700 10 client.17514 handle_command_reply: tid=0 2024-04-06T14:17:45.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.798+0000 7fa85c828b80 1 client.17514 shutdown 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 msgr2=0x555babc95800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 secure :-1 s=READY pgs=799 cs=0 l=1 rev1=1 crypto rx=0x7fa8400061e0 tx=0x7fa840007860 comp rx=0 tx=0).stop 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 shutdown_connections 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555babc905f0 0x555babc95800 unknown :-1 s=CLOSED pgs=799 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555babcf5540 0x555babcf7960 crc :-1 s=READY pgs=216 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555babc7d780 0x555babc96ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 --2- 172.21.15.90:0/1066907621 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555babc7e0a0 0x555babc952c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 >> 172.21.15.90:0/1066907621 conn(0x555babbdc6a0 msgr2=0x555babc8bca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa831ffb700 0 client.17514 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 shutdown_connections 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 1 -- 172.21.15.90:0/1066907621 wait complete. 2024-04-06T14:17:45.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.800+0000 7fa85c828b80 20 client.17514 trim_cache size 0 max 16384 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (397 inodes in the stack)", 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "f1207eab-ce7d-4b96-806c-58df855fad4d": { 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "f1207eab-ce7d-4b96-806c-58df855fad4d", 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:17:45.801 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:17:45.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 msgr2=0x7fa8340a3850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 0x7fa8340a3850 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fa840009c10 tx=0x7fa840009040 comp rx=0 tx=0).stop 2024-04-06T14:17:45.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 msgr2=0x7fa8481974c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:17:45.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 secure :-1 s=READY pgs=795 cs=0 l=1 rev1=1 crypto rx=0x7fa83800b480 tx=0x7fa83800b840 comp rx=0 tx=0).stop 2024-04-06T14:17:45.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 shutdown_connections 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa8340a1390 0x7fa8340a3850 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa84819c290 0x7fa84819c740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa848104700 0x7fa8481974c0 unknown :-1 s=CLOSED pgs=795 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 --2- 172.21.15.90:0/3036701791 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa848197a00 0x7fa84819bd50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 >> 172.21.15.90:0/3036701791 conn(0x7fa8480761b0 msgr2=0x7fa84810ac00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 shutdown_connections 2024-04-06T14:17:45.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:17:45.801+0000 7fa850674700 1 -- 172.21.15.90:0/3036701791 wait complete. 2024-04-06T14:17:45.816 INFO:tasks.cephfs.filesystem:scrub status for tag:f1207eab-ce7d-4b96-806c-58df855fad4d - {'path': '/', 'tag': 'f1207eab-ce7d-4b96-806c-58df855fad4d', 'options': 'recursive,force'} 2024-04-06T14:17:46.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:46 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:17:46.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:46 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:46.177 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:46 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:17:46.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:46 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:17:46.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:46 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:46.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:46 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:17:46.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:46 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:17:46.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:46 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:46.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:46 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:17:47.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:47 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:47.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:47 smithi105 ceph-mon[46969]: pgmap v386: 97 pgs: 97 active+clean; 6.4 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.17k op/s 2024-04-06T14:17:47.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:47 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:47.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:47 smithi090 ceph-mon[43420]: pgmap v386: 97 pgs: 97 active+clean; 6.4 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.17k op/s 2024-04-06T14:17:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:47 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:17:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:47 smithi134 ceph-mon[47549]: pgmap v386: 97 pgs: 97 active+clean; 6.4 GiB data, 20 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 18 MiB/s wr, 1.17k op/s 2024-04-06T14:17:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 720.0 s, 1373.2 tps, lat 73.417 ms stddev 90.902 2024-04-06T14:17:49.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:48 smithi105 ceph-mon[46969]: pgmap v387: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 21 MiB/s wr, 1.50k op/s 2024-04-06T14:17:49.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:48 smithi090 ceph-mon[43420]: pgmap v387: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 21 MiB/s wr, 1.50k op/s 2024-04-06T14:17:49.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:48 smithi134 ceph-mon[47549]: pgmap v387: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 21 MiB/s wr, 1.50k op/s 2024-04-06T14:17:51.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:51 smithi090 ceph-mon[43420]: pgmap v388: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 18 MiB/s wr, 1.42k op/s 2024-04-06T14:17:51.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:51 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:17:51.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:51 smithi134 ceph-mon[47549]: pgmap v388: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 18 MiB/s wr, 1.42k op/s 2024-04-06T14:17:51.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:51 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:17:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:51 smithi105 ceph-mon[46969]: pgmap v388: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 57 KiB/s rd, 18 MiB/s wr, 1.42k op/s 2024-04-06T14:17:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:51 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:17:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 725.0 s, 1744.6 tps, lat 56.823 ms stddev 66.744 2024-04-06T14:17:53.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:53 smithi090 ceph-mon[43420]: pgmap v389: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 333 KiB/s rd, 22 MiB/s wr, 1.66k op/s 2024-04-06T14:17:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:53 smithi134 ceph-mon[47549]: pgmap v389: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 333 KiB/s rd, 22 MiB/s wr, 1.66k op/s 2024-04-06T14:17:53.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:53 smithi105 ceph-mon[46969]: pgmap v389: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 333 KiB/s rd, 22 MiB/s wr, 1.66k op/s 2024-04-06T14:17:55.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:55 smithi134 ceph-mon[47549]: pgmap v390: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 316 KiB/s rd, 20 MiB/s wr, 1.52k op/s 2024-04-06T14:17:55.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:55 smithi105 ceph-mon[46969]: pgmap v390: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 316 KiB/s rd, 20 MiB/s wr, 1.52k op/s 2024-04-06T14:17:55.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:55 smithi090 ceph-mon[43420]: pgmap v390: 97 pgs: 97 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 316 KiB/s rd, 20 MiB/s wr, 1.52k op/s 2024-04-06T14:17:57.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:57 smithi134 ceph-mon[47549]: pgmap v391: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 317 KiB/s rd, 17 MiB/s wr, 1.22k op/s 2024-04-06T14:17:57.512 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:57 smithi105 ceph-mon[46969]: pgmap v391: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 317 KiB/s rd, 17 MiB/s wr, 1.22k op/s 2024-04-06T14:17:57.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:57 smithi090 ceph-mon[43420]: pgmap v391: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 317 KiB/s rd, 17 MiB/s wr, 1.22k op/s 2024-04-06T14:17:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 730.0 s, 1431.2 tps, lat 70.890 ms stddev 86.519 2024-04-06T14:17:59.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:17:58 smithi105 ceph-mon[46969]: pgmap v392: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 336 KiB/s rd, 17 MiB/s wr, 1.14k op/s 2024-04-06T14:17:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:17:58 smithi090 ceph-mon[43420]: pgmap v392: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 336 KiB/s rd, 17 MiB/s wr, 1.14k op/s 2024-04-06T14:17:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:17:58 smithi134 ceph-mon[47549]: pgmap v392: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 336 KiB/s rd, 17 MiB/s wr, 1.14k op/s 2024-04-06T14:18:01.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:01 smithi134 ceph-mon[47549]: pgmap v393: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 13 MiB/s wr, 797 op/s 2024-04-06T14:18:01.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:01 smithi105 ceph-mon[46969]: pgmap v393: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 13 MiB/s wr, 797 op/s 2024-04-06T14:18:01.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:01 smithi090 ceph-mon[43420]: pgmap v393: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 297 KiB/s rd, 13 MiB/s wr, 797 op/s 2024-04-06T14:18:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 735.0 s, 1079.2 tps, lat 80.669 ms stddev 83.235 2024-04-06T14:18:03.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:03 smithi134 ceph-mon[47549]: pgmap v394: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 14 MiB/s wr, 879 op/s 2024-04-06T14:18:03.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:03 smithi105 ceph-mon[46969]: pgmap v394: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 14 MiB/s wr, 879 op/s 2024-04-06T14:18:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:03 smithi090 ceph-mon[43420]: pgmap v394: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 315 KiB/s rd, 14 MiB/s wr, 879 op/s 2024-04-06T14:18:05.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:05 smithi134 ceph-mon[47549]: pgmap v395: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 13 MiB/s wr, 818 op/s 2024-04-06T14:18:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:05 smithi105 ceph-mon[46969]: pgmap v395: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 13 MiB/s wr, 818 op/s 2024-04-06T14:18:05.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:05 smithi090 ceph-mon[43420]: pgmap v395: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 13 MiB/s wr, 818 op/s 2024-04-06T14:18:07.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:07 smithi134 ceph-mon[47549]: pgmap v396: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 943 op/s 2024-04-06T14:18:07.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:07 smithi105 ceph-mon[46969]: pgmap v396: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 943 op/s 2024-04-06T14:18:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:07 smithi090 ceph-mon[43420]: pgmap v396: 97 pgs: 97 active+clean; 6.5 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 943 op/s 2024-04-06T14:18:08.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 740.0 s, 1547.5 tps, lat 73.291 ms stddev 112.428 2024-04-06T14:18:09.248 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:08 smithi105 ceph-mon[46969]: pgmap v397: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:18:09.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:08 smithi090 ceph-mon[43420]: pgmap v397: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:18:09.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:08 smithi134 ceph-mon[47549]: pgmap v397: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 19 MiB/s wr, 1.49k op/s 2024-04-06T14:18:11.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:11 smithi134 ceph-mon[47549]: pgmap v398: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 16 MiB/s wr, 1.32k op/s 2024-04-06T14:18:11.500 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:11 smithi105 ceph-mon[46969]: pgmap v398: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 16 MiB/s wr, 1.32k op/s 2024-04-06T14:18:11.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:11 smithi090 ceph-mon[43420]: pgmap v398: 97 pgs: 97 active+clean; 6.6 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 99 KiB/s rd, 16 MiB/s wr, 1.32k op/s 2024-04-06T14:18:12.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:12 smithi105 ceph-mon[46969]: osdmap e97: 12 total, 12 up, 12 in 2024-04-06T14:18:12.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:12 smithi090 ceph-mon[43420]: osdmap e97: 12 total, 12 up, 12 in 2024-04-06T14:18:12.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:12 smithi134 ceph-mon[47549]: osdmap e97: 12 total, 12 up, 12 in 2024-04-06T14:18:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 745.0 s, 1252.2 tps, lat 79.026 ms stddev 113.710 2024-04-06T14:18:13.504 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:13 smithi105 ceph-mon[46969]: pgmap v400: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 20 MiB/s wr, 1.53k op/s 2024-04-06T14:18:13.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:13 smithi090 ceph-mon[43420]: pgmap v400: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 20 MiB/s wr, 1.53k op/s 2024-04-06T14:18:13.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:13 smithi134 ceph-mon[47549]: pgmap v400: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.4 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 20 MiB/s wr, 1.53k op/s 2024-04-06T14:18:15.502 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:15 smithi105 ceph-mon[46969]: pgmap v401: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 16 MiB/s wr, 1.36k op/s 2024-04-06T14:18:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:15 smithi090 ceph-mon[43420]: pgmap v401: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 16 MiB/s wr, 1.36k op/s 2024-04-06T14:18:15.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:15 smithi134 ceph-mon[47549]: pgmap v401: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 6.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 119 KiB/s rd, 16 MiB/s wr, 1.36k op/s 2024-04-06T14:18:15.817 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:18:15.818 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- 172.21.15.90:0/534944269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae88104f70 msgr2=0x7fae8810caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 --2- 172.21.15.90:0/534944269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae88104f70 0x7fae8810caf0 secure :-1 s=READY pgs=800 cs=0 l=1 rev1=1 crypto rx=0x7fae8000b0a0 tx=0x7fae8000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- 172.21.15.90:0/534944269 shutdown_connections 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 --2- 172.21.15.90:0/534944269 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae88104f70 0x7fae8810caf0 unknown :-1 s=CLOSED pgs=800 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 --2- 172.21.15.90:0/534944269 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae880ffe70 0x7fae88100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 --2- 172.21.15.90:0/534944269 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fae88100820 0x7fae88104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- 172.21.15.90:0/534944269 >> 172.21.15.90:0/534944269 conn(0x7fae88076d50 msgr2=0x7fae88077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- 172.21.15.90:0/534944269 shutdown_connections 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- 172.21.15.90:0/534944269 wait complete. 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 Processor -- start 2024-04-06T14:18:16.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.376+0000 7fae900c6700 1 -- start start 2024-04-06T14:18:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 0x7fae88198700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fae88198c40 0x7fae881a3240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae881a3920 con 0x7fae880ffe70 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae881a3a90 con 0x7fae88104f70 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae900c6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae881a3c50 con 0x7fae88198c40 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40334/0 (socket says 172.21.15.90:40334) 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 -- 172.21.15.90:0/2518180565 learned_addr learned my addr 172.21.15.90:0/2518180565 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8d661700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 0x7fae88198700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 -- 172.21.15.90:0/2518180565 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fae88198c40 msgr2=0x7fae881a3240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fae88198c40 0x7fae881a3240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 -- 172.21.15.90:0/2518180565 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 msgr2=0x7fae88198700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 0x7fae88198700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 -- 172.21.15.90:0/2518180565 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae80009d00 con 0x7fae880ffe70 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8d661700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 0x7fae88198700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:16.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae8de62700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7fae8400e9a0 tx=0x7fae8400ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.377+0000 7fae7effd700 1 -- 172.21.15.90:0/2518180565 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae84009790 con 0x7fae880ffe70 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.378+0000 7fae7effd700 1 -- 172.21.15.90:0/2518180565 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fae8400cbd0 con 0x7fae880ffe70 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.378+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae88196010 con 0x7fae880ffe70 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.378+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae88196520 con 0x7fae880ffe70 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.380+0000 7fae7effd700 1 -- 172.21.15.90:0/2518180565 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae84018680 con 0x7fae880ffe70 2024-04-06T14:18:16.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.381+0000 7fae7effd700 1 -- 172.21.15.90:0/2518180565 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fae840188a0 con 0x7fae880ffe70 2024-04-06T14:18:16.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.381+0000 7fae7effd700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 0x7fae740a39a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 Processor -- start 2024-04-06T14:18:16.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae7effd700 1 -- 172.21.15.90:0/2518180565 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fae84014070 con 0x7fae880ffe70 2024-04-06T14:18:16.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae8d661700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 0x7fae740a39a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 -- start start 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e227b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 0x56097e2318a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e17f990 con 0x56097e227780 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.387+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e17d350 con 0x56097e231de0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.387+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e178c40 con 0x56097e2280a0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.382+0000 7fae8e663700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.383+0000 7fae8d661700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 0x7fae740a39a0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fae88068610 tx=0x7fae7800f040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.383+0000 7fae8de62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e227b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8e663700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:58974/0 (socket says 172.21.15.90:58974) 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8e663700 1 -- 172.21.15.90:0/3389726014 learned_addr learned my addr 172.21.15.90:0/3389726014 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.387+0000 7fae8d661700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 0x56097e2318a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 -- 172.21.15.90:0/3389726014 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 msgr2=0x56097e2341d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 -- 172.21.15.90:0/3389726014 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 msgr2=0x56097e227b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e227b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 -- 172.21.15.90:0/3389726014 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56097dffa3b0 con 0x56097e2280a0 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8e663700 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae8d661700 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 0x56097e2318a0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fae7800dfd0 tx=0x7fae78011000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6634618ad90fc29d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae73fff700 1 -- 172.21.15.90:0/3389726014 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae7801b840 con 0x56097e2280a0 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae73fff700 1 -- 172.21.15.90:0/3389726014 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fae7801be60 con 0x56097e2280a0 2024-04-06T14:18:16.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.388+0000 7fae73fff700 1 -- 172.21.15.90:0/3389726014 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae7801a440 con 0x56097e2280a0 2024-04-06T14:18:16.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3389726014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 msgr2=0x56097e2318a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 0x56097e2318a0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fae7800dfd0 tx=0x7fae78011000 comp rx=0 tx=0).stop 2024-04-06T14:18:16.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3389726014 shutdown_connections 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e2341d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e227b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3389726014 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e2280a0 0x56097e2318a0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3389726014 >> 172.21.15.90:0/3389726014 conn(0x56097e1866a0 msgr2=0x56097e222a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.389+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3389726014 shutdown_connections 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3389726014 wait complete. 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 -- start start 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 Processor -- start 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e2316d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e22ef90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e114d80 con 0x56097e2280a0 2024-04-06T14:18:16.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e114a20 con 0x56097e231de0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097dffc6a0 con 0x56097e227780 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae9c27ab80 10 client.0 _ll_register_callbacks cb 0x56097e11e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae8d661700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae8d661700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40358/0 (socket says 172.21.15.90:40358) 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.390+0000 7fae8d661700 1 -- 172.21.15.90:0/3542697702 learned_addr learned my addr 172.21.15.90:0/3542697702 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8e663700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e22ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8de62700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e2316d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 -- 172.21.15.90:0/3542697702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 msgr2=0x56097e2316d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e2316d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 -- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 msgr2=0x56097e22ef90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e22ef90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 -- 172.21.15.90:0/3542697702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae8810d360 con 0x56097e2280a0 2024-04-06T14:18:16.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae8d661700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7fae7800f010 tx=0x7fae78006d90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=25e9d2211c1e4aba server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae717fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae7801ba50 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fae78018850 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56097e139380 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fae7801bbb0 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae9c27ab80 10 client.17526 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.391+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fae78012030 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56097e139380 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fae7801a440 con 0x56097e2280a0 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 10 client.17526 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 10 client.17526 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 20 client.17526 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:16.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e263990 0x56097e265db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 4 client.17526 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:16.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.392+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56097e171700 con 0x56097e263990 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.393+0000 7fae8de62700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e263990 0x56097e265db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.393+0000 7fae717fa700 10 client.17526 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.393+0000 7fae8de62700 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e263990 0x56097e265db0 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e98b536c4dfa22e9 server_cookie=3827714e6a62a917 in_seq=0 out_seq=0 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae717fa700 1 -- 172.21.15.90:0/3542697702 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x56097e171700 con 0x56097e263990 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae717fa700 10 client.17526 handle_command_reply: tid=0 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 client.17526 shutdown 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 msgr2=0x56097e22ea10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 secure :-1 s=READY pgs=802 cs=0 l=1 rev1=1 crypto rx=0x7fae7800f010 tx=0x7fae78006d90 comp rx=0 tx=0).stop 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 shutdown_connections 2024-04-06T14:18:16.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e231de0 0x56097e22ef90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e263990 0x56097e265db0 crc :-1 s=READY pgs=217 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e2316d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/3542697702 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e2280a0 0x56097e22ea10 unknown :-1 s=CLOSED pgs=802 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 >> 172.21.15.90:0/3542697702 conn(0x56097e1866a0 msgr2=0x56097e221c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae717fa700 0 client.17526 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 shutdown_connections 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 1 -- 172.21.15.90:0/3542697702 wait complete. 2024-04-06T14:18:16.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.394+0000 7fae9c27ab80 20 client.17526 trim_cache size 0 max 16384 2024-04-06T14:18:16.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.415+0000 7fae9c27ab80 1 Processor -- start 2024-04-06T14:18:16.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.415+0000 7fae9c27ab80 1 -- start start 2024-04-06T14:18:16.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.415+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e249b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.415+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 0x56097e229d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e228a10 con 0x56097e231de0 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e223650 con 0x56097e2280a0 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e229880 con 0x56097e227780 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8e663700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 0x56097e229d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8e663700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40362/0 (socket says 172.21.15.90:40362) 2024-04-06T14:18:16.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8e663700 1 -- 172.21.15.90:0/1056669222 learned_addr learned my addr 172.21.15.90:0/1056669222 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8de62700 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e249b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 -- 172.21.15.90:0/1056669222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 msgr2=0x56097e249b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e249b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 -- 172.21.15.90:0/1056669222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 msgr2=0x56097e22a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 -- 172.21.15.90:0/1056669222 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56097dfa6b60 con 0x56097e2280a0 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8e663700 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae8d661700 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 0x56097e229d40 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fae7800dd80 tx=0x7fae78018f20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5488a02fd9783b2f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae7cff9700 1 -- 172.21.15.90:0/1056669222 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae7801a970 con 0x56097e2280a0 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae7cff9700 1 -- 172.21.15.90:0/1056669222 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fae780186d0 con 0x56097e2280a0 2024-04-06T14:18:16.419 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.416+0000 7fae7cff9700 1 -- 172.21.15.90:0/1056669222 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae78005ea0 con 0x56097e2280a0 2024-04-06T14:18:16.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 -- 172.21.15.90:0/1056669222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 msgr2=0x56097e229d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 0x56097e229d40 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fae7800dd80 tx=0x7fae78018f20 comp rx=0 tx=0).stop 2024-04-06T14:18:16.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 -- 172.21.15.90:0/1056669222 shutdown_connections 2024-04-06T14:18:16.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e231de0 0x56097e22a280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e227780 0x56097e249b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/1056669222 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e2280a0 0x56097e229d40 secure :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7fae7800dd80 tx=0x7fae78018f20 comp rx=0 tx=0).stop 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.417+0000 7fae9c27ab80 1 -- 172.21.15.90:0/1056669222 >> 172.21.15.90:0/1056669222 conn(0x56097e1866a0 msgr2=0x56097e24e250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.419+0000 7fae9c27ab80 1 -- 172.21.15.90:0/1056669222 shutdown_connections 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 -- 172.21.15.90:0/1056669222 wait complete. 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 -- start start 2024-04-06T14:18:16.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 Processor -- start 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e231de0 0x56097e248020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 0x56097e2b6780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.420+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e2b6e40 con 0x56097e227780 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e2b6fb0 con 0x56097e248670 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x56097e2b7170 con 0x56097e231de0 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 10 client.0 _ll_register_callbacks cb 0x56097e11e470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:16.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8e663700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 0x56097e2b6780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40366/0 (socket says 172.21.15.90:40366) 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 -- 172.21.15.90:0/2731379262 learned_addr learned my addr 172.21.15.90:0/2731379262 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8d661700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e231de0 0x56097e248020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 -- 172.21.15.90:0/2731379262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e231de0 msgr2=0x56097e248020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e231de0 0x56097e248020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 -- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 msgr2=0x56097e2b6780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 0x56097e2b6780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 -- 172.21.15.90:0/2731379262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae78012030 con 0x56097e227780 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8e663700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 0x56097e2b6780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae8de62700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7fae84010c90 tx=0x7fae84097490 comp rx=0 tx=0).ready entity=mon.0 client_cookie=282a7e00dac35a24 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae717fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:16.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae8409b020 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7fae840b92c0 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fae7801be60 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fae8409b280 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 10 client.17532 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.421+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x56097e13eea0 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fae7801be60 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fae840b3bd0 con 0x56097e227780 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 10 client.17532 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 10 client.17532 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 20 client.17532 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e2b8900 0x56097e2bad20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 4 client.17532 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56097dfc7cf0 con 0x56097e2b8900 2024-04-06T14:18:16.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.422+0000 7fae8d661700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e2b8900 0x56097e2bad20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae8d661700 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e2b8900 0x56097e2bad20 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ff4dc23e16e3f995 server_cookie=13d31f96c431bca3 in_seq=0 out_seq=0 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae717fa700 10 client.17532 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae717fa700 1 -- 172.21.15.90:0/2731379262 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x56097dfc7cf0 con 0x56097e2b8900 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae717fa700 10 client.17532 handle_command_reply: tid=0 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 client.17532 shutdown 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 msgr2=0x56097e247ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 secure :-1 s=READY pgs=803 cs=0 l=1 rev1=1 crypto rx=0x7fae84010c90 tx=0x7fae84097490 comp rx=0 tx=0).stop 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 shutdown_connections 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x56097e248670 0x56097e2b6780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x56097e227780 0x56097e247ae0 unknown :-1 s=CLOSED pgs=803 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x56097e2b8900 0x56097e2bad20 crc :-1 s=READY pgs=218 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 --2- 172.21.15.90:0/2731379262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x56097e231de0 0x56097e248020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae717fa700 0 client.17532 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 >> 172.21.15.90:0/2731379262 conn(0x56097e1866a0 msgr2=0x56097e229d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 shutdown_connections 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 1 -- 172.21.15.90:0/2731379262 wait complete. 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.423+0000 7fae9c27ab80 20 client.17532 trim_cache size 0 max 16384 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:18:16.429 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:16.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 msgr2=0x7fae740a39a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 0x7fae740a39a0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fae88068610 tx=0x7fae7800f040 comp rx=0 tx=0).stop 2024-04-06T14:18:16.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 msgr2=0x7fae881981c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 secure :-1 s=READY pgs=801 cs=0 l=1 rev1=1 crypto rx=0x7fae8400e9a0 tx=0x7fae8400ecb0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 shutdown_connections 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fae740a14e0 0x7fae740a39a0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fae88198c40 0x7fae881a3240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fae880ffe70 0x7fae881981c0 unknown :-1 s=CLOSED pgs=801 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 --2- 172.21.15.90:0/2518180565 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fae88104f70 0x7fae88198700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 >> 172.21.15.90:0/2518180565 conn(0x7fae88076d50 msgr2=0x7fae880fdd50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 shutdown_connections 2024-04-06T14:18:16.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.424+0000 7fae900c6700 1 -- 172.21.15.90:0/2518180565 wait complete. 2024-04-06T14:18:16.438 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:18:16.439 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:18:16.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.609+0000 7ffb0eae6700 1 -- 172.21.15.90:0/2232591076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ffd60 msgr2=0x7ffb08105890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.609+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/2232591076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ffd60 0x7ffb08105890 secure :-1 s=READY pgs=804 cs=0 l=1 rev1=1 crypto rx=0x7ffaf4009990 tx=0x7ffaf4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 -- 172.21.15.90:0/2232591076 shutdown_connections 2024-04-06T14:18:16.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/2232591076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb0810db40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/2232591076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb080ff3b0 0x7ffb080ff790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/2232591076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ffd60 0x7ffb08105890 unknown :-1 s=CLOSED pgs=804 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 -- 172.21.15.90:0/2232591076 >> 172.21.15.90:0/2232591076 conn(0x7ffb080760a0 msgr2=0x7ffb080fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:16.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 -- 172.21.15.90:0/2232591076 shutdown_connections 2024-04-06T14:18:16.612 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 -- 172.21.15.90:0/2232591076 wait complete. 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 Processor -- start 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 -- start start 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb08115480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.611+0000 7ffb0eae6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb08110080 0x7ffb08110530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0eae6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb08110c40 con 0x7ffb080ff3b0 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0eae6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb08110db0 con 0x7ffb08110080 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0eae6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb0811aca0 con 0x7ffb08105f70 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.614 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40382/0 (socket says 172.21.15.90:40382) 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 -- 172.21.15.90:0/46168025 learned_addr learned my addr 172.21.15.90:0/46168025 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb07fff700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb08115480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0d083700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb08110080 0x7ffb08110530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 -- 172.21.15.90:0/46168025 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 msgr2=0x7ffb08115480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb08115480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 -- 172.21.15.90:0/46168025 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb08110080 msgr2=0x7ffb08110530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb08110080 0x7ffb08110530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.612+0000 7ffb0c882700 1 -- 172.21.15.90:0/46168025 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffaf40095f0 con 0x7ffb080ff3b0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb07fff700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb08115480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb0c882700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7ffafc00e9a0 tx=0x7ffafc00ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffafc00cbd0 con 0x7ffb080ff3b0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffafc00cd30 con 0x7ffb080ff3b0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb0811af80 con 0x7ffb080ff3b0 2024-04-06T14:18:16.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffafc018710 con 0x7ffb080ff3b0 2024-04-06T14:18:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.613+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb0811b4e0 con 0x7ffb080ff3b0 2024-04-06T14:18:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.614+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffb08109200 con 0x7ffb080ff3b0 2024-04-06T14:18:16.616 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.616+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ffafc010980 con 0x7ffb080ff3b0 2024-04-06T14:18:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.617+0000 7ffb05ffb700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 0x7ffaf80a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:16.617 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.617+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7ffafc014070 con 0x7ffb080ff3b0 2024-04-06T14:18:16.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.619+0000 7ffb07fff700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 0x7ffaf80a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:16.619 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.619+0000 7ffb07fff700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 0x7ffaf80a3810 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7ffaf4000c00 tx=0x7ffaf4005c00 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:16.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:16.622+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ffafc0968e0 con 0x7ffb080ff3b0 2024-04-06T14:18:17.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.013+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ffb08102460 con 0x7ffb080ff3b0 2024-04-06T14:18:17.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.014+0000 7ffb05ffb700 1 -- 172.21.15.90:0/46168025 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7ffafc099f00 con 0x7ffb080ff3b0 2024-04-06T14:18:17.015 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:18:17.015 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:18:17.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 msgr2=0x7ffaf80a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 0x7ffaf80a3810 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7ffaf4000c00 tx=0x7ffaf4005c00 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 msgr2=0x7ffb08114f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 secure :-1 s=READY pgs=805 cs=0 l=1 rev1=1 crypto rx=0x7ffafc00e9a0 tx=0x7ffafc00ed60 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 shutdown_connections 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ffaf80a1350 0x7ffaf80a3810 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ffb08110080 0x7ffb08110530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ffb080ff3b0 0x7ffb08114f20 unknown :-1 s=CLOSED pgs=805 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 --2- 172.21.15.90:0/46168025 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ffb08105f70 0x7ffb08115480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 >> 172.21.15.90:0/46168025 conn(0x7ffb080760a0 msgr2=0x7ffb08104290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 shutdown_connections 2024-04-06T14:18:17.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.017+0000 7ffb0eae6700 1 -- 172.21.15.90:0/46168025 wait complete. 2024-04-06T14:18:17.021 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:18:17.036 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:17.036 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:18:17.295 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:17 smithi090 ceph-mon[43420]: pgmap v402: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:18:17.295 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:17 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/46168025' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 -- 172.21.15.90:0/605535277 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 msgr2=0x7f3c4c107810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 --2- 172.21.15.90:0/605535277 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c107810 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7f3c400098f0 tx=0x7f3c40009c00 comp rx=0 tx=0).stop 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 -- 172.21.15.90:0/605535277 shutdown_connections 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 --2- 172.21.15.90:0/605535277 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 0x7f3c4c1141f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 --2- 172.21.15.90:0/605535277 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c4c10d450 0x7f3c4c10d830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 --2- 172.21.15.90:0/605535277 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c107810 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 -- 172.21.15.90:0/605535277 >> 172.21.15.90:0/605535277 conn(0x7f3c4c076ba0 msgr2=0x7f3c4c105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 -- 172.21.15.90:0/605535277 shutdown_connections 2024-04-06T14:18:17.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.413+0000 7f3c51d40700 1 -- 172.21.15.90:0/605535277 wait complete. 2024-04-06T14:18:17.415 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 Processor -- start 2024-04-06T14:18:17.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 -- start start 2024-04-06T14:18:17.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 0x7f3c4c1a3a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.416 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c4c10d450 0x7f3c4c19d6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c4c19dd30 con 0x7f3c4c10d450 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c4c19dea0 con 0x7f3c4c107390 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c51d40700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c4c19e060 con 0x7f3c4c107d50 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c4b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c4b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59022/0 (socket says 172.21.15.90:59022) 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.414+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/2910334872 learned_addr learned my addr 172.21.15.90:0/2910334872 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4affd700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 0x7f3c4c1a3a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/2910334872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 msgr2=0x7f3c4c1a3a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 0x7f3c4c1a3a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c4c10d450 msgr2=0x7f3c4c19d6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c4c10d450 0x7f3c4c19d6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/2910334872 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c3c0095f0 con 0x7f3c4c107390 2024-04-06T14:18:17.417 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c00ec10 tx=0x7f3c3c00c2d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.415+0000 7f3c48ff9700 1 -- 172.21.15.90:0/2910334872 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c3c009a10 con 0x7f3c4c107390 2024-04-06T14:18:17.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.416+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c40009520 con 0x7f3c4c107390 2024-04-06T14:18:17.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.416+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c4c19e530 con 0x7f3c4c107390 2024-04-06T14:18:17.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.416+0000 7f3c48ff9700 1 -- 172.21.15.90:0/2910334872 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c3c0044e0 con 0x7f3c4c107390 2024-04-06T14:18:17.418 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.416+0000 7f3c48ff9700 1 -- 172.21.15.90:0/2910334872 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c3c010430 con 0x7f3c4c107390 2024-04-06T14:18:17.420 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.419+0000 7f3c5def4b80 1 Processor -- start 2024-04-06T14:18:17.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 -- start start 2024-04-06T14:18:17.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e2780 0x55c6e31e2b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31e30a0 0x55c6e31ec8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.421 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e313a990 con 0x55c6e31ecde0 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e3138350 con 0x55c6e31e2780 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e3133c40 con 0x55c6e31e30a0 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c48ff9700 1 -- 172.21.15.90:0/2910334872 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3c3c004020 con 0x7f3c4c107390 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c4affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40412/0 (socket says 172.21.15.90:40412) 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c4affd700 1 -- 172.21.15.90:0/290687406 learned_addr learned my addr 172.21.15.90:0/290687406 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.420+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e2780 0x55c6e31e2b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c48ff9700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 0x7f3c340a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 -- 172.21.15.90:0/290687406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31e30a0 msgr2=0x55c6e31ec8a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31e30a0 0x55c6e31ec8a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 -- 172.21.15.90:0/290687406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e2780 msgr2=0x55c6e31e2b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.422 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e2780 0x55c6e31e2b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 -- 172.21.15.90:0/290687406 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c6e2fb53b0 con 0x55c6e31ecde0 2024-04-06T14:18:17.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c48ff9700 1 -- 172.21.15.90:0/2910334872 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f3c3c014070 con 0x7f3c4c107390 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.421+0000 7f3c4affd700 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7f3c400098f0 tx=0x7f3c40009480 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c3508101274c261f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c31ffb700 1 -- 172.21.15.90:0/290687406 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c4001c040 con 0x55c6e31ecde0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c31ffb700 1 -- 172.21.15.90:0/290687406 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c4000f920 con 0x55c6e31ecde0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c31ffb700 1 -- 172.21.15.90:0/290687406 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c400052a0 con 0x55c6e31ecde0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 0x7f3c340a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c5def4b80 1 -- 172.21.15.90:0/290687406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 msgr2=0x55c6e31ef1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 0x7f3c340a37d0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f3c44009fd0 tx=0x7f3c44009040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.422+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 secure :-1 s=READY pgs=806 cs=0 l=1 rev1=1 crypto rx=0x7f3c400098f0 tx=0x7f3c40009480 comp rx=0 tx=0).stop 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 -- 172.21.15.90:0/290687406 shutdown_connections 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31ecde0 0x55c6e31ef1d0 unknown :-1 s=CLOSED pgs=806 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e2780 0x55c6e31e2b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/290687406 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31e30a0 0x55c6e31ec8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 -- 172.21.15.90:0/290687406 >> 172.21.15.90:0/290687406 conn(0x55c6e31416a0 msgr2=0x55c6e31dda80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 -- 172.21.15.90:0/290687406 shutdown_connections 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 -- 172.21.15.90:0/290687406 wait complete. 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 -- start start 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.423+0000 7f3c5def4b80 1 Processor -- start 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e31ec7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e31e9ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e30cfd80 con 0x55c6e31e2780 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e30cfa20 con 0x55c6e31e30a0 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e2fb76a0 con 0x55c6e31ecde0 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c5def4b80 10 client.0 _ll_register_callbacks cb 0x55c6e30d9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:17.426 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44224/0 (socket says 172.21.15.90:44224) 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 -- 172.21.15.90:0/672297929 learned_addr learned my addr 172.21.15.90:0/672297929 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e31ec7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 -- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 msgr2=0x55c6e31e9ad0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e31e9ad0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 -- 172.21.15.90:0/672297929 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 msgr2=0x55c6e31ec7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e31ec7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.427 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.424+0000 7f3c4bfff700 1 -- 172.21.15.90:0/672297929 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c4c114ae0 con 0x55c6e31ecde0 2024-04-06T14:18:17.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.426+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3c44015b10 tx=0x7f3c4400b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b68a7df3a27ecb51 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.426+0000 7f3c2affd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:18:17.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.427+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c4400e030 con 0x55c6e31ecde0 2024-04-06T14:18:17.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.427+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f3c4401d040 con 0x55c6e31ecde0 2024-04-06T14:18:17.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.427+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c6e30f4380 con 0x55c6e31ecde0 2024-04-06T14:18:17.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3c440091d0 con 0x55c6e31ecde0 2024-04-06T14:18:17.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 10 client.27037 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:17.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c6e2f61b60 con 0x55c6e31ecde0 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c6e30f4380 con 0x55c6e31ecde0 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3c44021bf0 con 0x55c6e31ecde0 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 10 client.27037 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 10 client.27037 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 20 client.27037 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e320da60 0x55c6e320fe80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 4 client.27037 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c6e312c700 con 0x55c6e320da60 2024-04-06T14:18:17.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.431+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e320da60 0x55c6e320fe80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.432+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e320da60 0x55c6e320fe80 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=c50685f8ca743deb server_cookie=e04d1be14b91ea40 in_seq=0 out_seq=0 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.432+0000 7f3c2affd700 10 client.27037 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.432+0000 7f3c2affd700 1 -- 172.21.15.90:0/672297929 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c6e312c700 con 0x55c6e320da60 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.432+0000 7f3c2affd700 10 client.27037 handle_command_reply: tid=0 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.432+0000 7f3c5def4b80 1 client.27037 shutdown 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 msgr2=0x55c6e31ea150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3c44015b10 tx=0x7f3c4400b040 comp rx=0 tx=0).stop 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 shutdown_connections 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e320da60 0x55c6e320fe80 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31ea150 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e31ec7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/672297929 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e31e9ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 >> 172.21.15.90:0/672297929 conn(0x55c6e31416a0 msgr2=0x55c6e31dcc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c2affd700 0 client.27037 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 shutdown_connections 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.433+0000 7f3c5def4b80 1 -- 172.21.15.90:0/672297929 wait complete. 2024-04-06T14:18:17.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.434+0000 7f3c5def4b80 20 client.27037 trim_cache size 0 max 16384 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 Processor -- start 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 -- start start 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e3204aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e3209900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31dd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e31df260 con 0x55c6e31e2780 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e320a5b0 con 0x55c6e31e30a0 2024-04-06T14:18:17.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.450+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e31de6d0 con 0x55c6e31ecde0 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e3209900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e3209900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59036/0 (socket says 172.21.15.90:59036) 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4affd700 1 -- 172.21.15.90:0/1448744151 learned_addr learned my addr 172.21.15.90:0/1448744151 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e3204aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4bfff700 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31dd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/1448744151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 msgr2=0x55c6e31dd3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31dd3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/1448744151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 msgr2=0x55c6e3209900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e3209900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/1448744151 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c6e30f9ea0 con 0x55c6e31e2780 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e3204aa0 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c0ca940 tx=0x7f3c3c000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84478768d6f9f0ef server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.451+0000 7f3c327fc700 1 -- 172.21.15.90:0/1448744151 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c3c01f340 con 0x55c6e31e2780 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c327fc700 1 -- 172.21.15.90:0/1448744151 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c3c09aa00 con 0x55c6e31e2780 2024-04-06T14:18:17.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c327fc700 1 -- 172.21.15.90:0/1448744151 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c3c098440 con 0x55c6e31e2780 2024-04-06T14:18:17.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 -- 172.21.15.90:0/1448744151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 msgr2=0x55c6e3204aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e3204aa0 secure :-1 s=READY pgs=807 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c0ca940 tx=0x7f3c3c000bc0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 -- 172.21.15.90:0/1448744151 shutdown_connections 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e31dd3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e3204aa0 unknown :-1 s=CLOSED pgs=807 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/1448744151 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e31e30a0 0x55c6e3209900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.452+0000 7f3c5def4b80 1 -- 172.21.15.90:0/1448744151 >> 172.21.15.90:0/1448744151 conn(0x55c6e31416a0 msgr2=0x55c6e32091f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.453+0000 7f3c5def4b80 1 -- 172.21.15.90:0/1448744151 shutdown_connections 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 -- 172.21.15.90:0/1448744151 wait complete. 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 -- start start 2024-04-06T14:18:17.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 Processor -- start 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e3204820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e32028d0 0x55c6e3202d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e32032d0 con 0x55c6e31e2780 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e3203410 con 0x55c6e32028d0 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c6e32035d0 con 0x55c6e31ecde0 2024-04-06T14:18:17.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.454+0000 7f3c5def4b80 10 client.0 _ll_register_callbacks cb 0x55c6e30d9470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40446/0 (socket says 172.21.15.90:40446) 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/941610693 learned_addr learned my addr 172.21.15.90:0/941610693 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4affd700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e3204820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/941610693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 msgr2=0x55c6e3204820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e3204820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e32028d0 msgr2=0x55c6e3202d50 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e32028d0 0x55c6e3202d50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 -- 172.21.15.90:0/941610693 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c4c19e530 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4affd700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e3204820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c4b7fe700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c0a2250 tx=0x7f3c3c003bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dab27f6c0ed08a09 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c2affd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.456+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c3c0a2780 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f3c3c00cde0 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3c4000f920 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3c3c0986a0 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c5def4b80 10 client.17553 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c6e2fb9cd0 con 0x55c6e31e2780 2024-04-06T14:18:17.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.457+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c4000f920 con 0x55c6e31e2780 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3c3c09a560 con 0x55c6e31e2780 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 10 client.17553 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 10 client.17553 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 20 client.17553 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e3273b20 0x55c6e3275f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 4 client.17553 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c6e2f82cf0 con 0x55c6e3273b20 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c4affd700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e3273b20 0x55c6e3275f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c4affd700 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e3273b20 0x55c6e3275f40 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=fdabfa7e54e8fa68 server_cookie=eb88676ae25d281 in_seq=0 out_seq=0 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c2affd700 10 client.17553 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c2affd700 1 -- 172.21.15.90:0/941610693 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c6e2f82cf0 con 0x55c6e3273b20 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c2affd700 10 client.17553 handle_command_reply: tid=0 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.458+0000 7f3c5def4b80 1 client.17553 shutdown 2024-04-06T14:18:17.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 msgr2=0x55c6e32042e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 secure :-1 s=READY pgs=808 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c0a2250 tx=0x7f3c3c003bc0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 shutdown_connections 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55c6e3273b20 0x55c6e3275f40 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c6e32028d0 0x55c6e3202d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c6e31e2780 0x55c6e32042e0 unknown :-1 s=CLOSED pgs=808 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 --2- 172.21.15.90:0/941610693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c6e31ecde0 0x55c6e3204820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 >> 172.21.15.90:0/941610693 conn(0x55c6e31416a0 msgr2=0x55c6e31eb370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.462 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.459+0000 7f3c2affd700 0 client.17553 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:17.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.460+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 shutdown_connections 2024-04-06T14:18:17.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.460+0000 7f3c5def4b80 1 -- 172.21.15.90:0/941610693 wait complete. 2024-04-06T14:18:17.463 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.460+0000 7f3c5def4b80 20 client.17553 trim_cache size 0 max 16384 2024-04-06T14:18:17.464 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 msgr2=0x7f3c340a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 0x7f3c340a37d0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f3c44009fd0 tx=0x7f3c44009040 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 msgr2=0x7f3c4c1a3550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c00ec10 tx=0x7f3c3c00c2d0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 shutdown_connections 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3c340a1310 0x7f3c340a37d0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3c4c10d450 0x7f3c4c19d6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c4c107390 0x7f3c4c1a3550 secure :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7f3c3c00ec10 tx=0x7f3c3c00c2d0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 --2- 172.21.15.90:0/2910334872 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3c4c107d50 0x7f3c4c1a3a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.461+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 >> 172.21.15.90:0/2910334872 conn(0x7f3c4c076ba0 msgr2=0x7f3c4c0780b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.463+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 shutdown_connections 2024-04-06T14:18:17.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.463+0000 7f3c51d40700 1 -- 172.21.15.90:0/2910334872 wait complete. 2024-04-06T14:18:17.480 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:17.480 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:18:17.508 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:17 smithi105 ceph-mon[46969]: pgmap v402: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:18:17.508 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:17 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/46168025' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:17.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:17 smithi134 ceph-mon[47549]: pgmap v402: 97 pgs: 5 active+clean+snaptrim, 10 active+clean+snaptrim_wait, 82 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 15 MiB/s wr, 1.15k op/s 2024-04-06T14:18:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:17 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/46168025' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:17.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 -- 172.21.15.90:0/2111488181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f230010d450 msgr2=0x7f230010d830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 --2- 172.21.15.90:0/2111488181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f230010d450 0x7f230010d830 secure :-1 s=READY pgs=809 cs=0 l=1 rev1=1 crypto rx=0x7f22f4009990 tx=0x7f22f4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 -- 172.21.15.90:0/2111488181 shutdown_connections 2024-04-06T14:18:17.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 --2- 172.21.15.90:0/2111488181 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 0x7f23001141f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 --2- 172.21.15.90:0/2111488181 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f230010d450 0x7f230010d830 unknown :-1 s=CLOSED pgs=809 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 --2- 172.21.15.90:0/2111488181 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2300107390 0x7f2300107810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 -- 172.21.15.90:0/2111488181 >> 172.21.15.90:0/2111488181 conn(0x7f2300076ba0 msgr2=0x7f2300105bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 -- 172.21.15.90:0/2111488181 shutdown_connections 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.759+0000 7f2306ac9700 1 -- 172.21.15.90:0/2111488181 wait complete. 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 Processor -- start 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 -- start start 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 0x7f2300081700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f230010d450 0x7f230007a2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f230007aa80 con 0x7f2300107390 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f230007abf0 con 0x7f230010d450 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.760+0000 7f2306ac9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f230007adb0 con 0x7f2300107d50 2024-04-06T14:18:17.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40458/0 (socket says 172.21.15.90:40458) 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 -- 172.21.15.90:0/1679069595 learned_addr learned my addr 172.21.15.90:0/1679069595 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f22fffff700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 0x7f2300081700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 -- 172.21.15.90:0/1679069595 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 msgr2=0x7f2300081700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 0x7f2300081700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 -- 172.21.15.90:0/1679069595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f230010d450 msgr2=0x7f230007a2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f230010d450 0x7f230007a2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 -- 172.21.15.90:0/1679069595 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22f40095f0 con 0x7f2300107390 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.761+0000 7f2304865700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7f22f400c010 tx=0x7f22f40053a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.762+0000 7f22fdffb700 1 -- 172.21.15.90:0/1679069595 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f401d070 con 0x7f2300107390 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.762+0000 7f22fdffb700 1 -- 172.21.15.90:0/1679069595 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22f4003c80 con 0x7f2300107390 2024-04-06T14:18:17.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.762+0000 7f22fdffb700 1 -- 172.21.15.90:0/1679069595 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f400fdc0 con 0x7f2300107390 2024-04-06T14:18:17.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.763+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f230007b030 con 0x7f2300107390 2024-04-06T14:18:17.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.763+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f230007ef50 con 0x7f2300107390 2024-04-06T14:18:17.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.767+0000 7f22fdffb700 1 -- 172.21.15.90:0/1679069595 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f22f400f4f0 con 0x7f2300107390 2024-04-06T14:18:17.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.768+0000 7f22fdffb700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 0x7f22f00a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.768+0000 7f22fdffb700 1 -- 172.21.15.90:0/1679069595 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f22f40cdee0 con 0x7f2300107390 2024-04-06T14:18:17.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.769+0000 7f22fffff700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 0x7f22f00a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.769+0000 7f22fffff700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 0x7f22f00a38e0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f22ec006fd0 tx=0x7f22ec008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.773+0000 7f2312c7db80 1 Processor -- start 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.774+0000 7f2312c7db80 1 -- start start 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.774+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815b0b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.774+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815ba8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.774+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2312c7db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b881508990 con 0x55b8815b10a0 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2312c7db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b881506350 con 0x55b8815bade0 2024-04-06T14:18:17.775 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2312c7db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b881501c40 con 0x55b8815b0780 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2305066700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2305066700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59052/0 (socket says 172.21.15.90:59052) 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.775+0000 7f2305066700 1 -- 172.21.15.90:0/2798294341 learned_addr learned my addr 172.21.15.90:0/2798294341 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 -- 172.21.15.90:0/2798294341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 msgr2=0x55b8815b0b60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815b0b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f22fffff700 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815ba8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 -- 172.21.15.90:0/2798294341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 msgr2=0x55b8815ba8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.776 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815ba8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 -- 172.21.15.90:0/2798294341 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b8813833b0 con 0x55b8815bade0 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f22fffff700 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815ba8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.776+0000 7f2305066700 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f22f8009d00 tx=0x7f22f800b330 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bd58fa71c3e38a5e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.777+0000 7f22eaffd700 1 -- 172.21.15.90:0/2798294341 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f800e070 con 0x55b8815bade0 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.777+0000 7f22eaffd700 1 -- 172.21.15.90:0/2798294341 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22f8009830 con 0x55b8815bade0 2024-04-06T14:18:17.777 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.777+0000 7f22eaffd700 1 -- 172.21.15.90:0/2798294341 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f8011390 con 0x55b8815bade0 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 -- 172.21.15.90:0/2798294341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 msgr2=0x55b8815bd1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7f22f8009d00 tx=0x7f22f800b330 comp rx=0 tx=0).stop 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 -- 172.21.15.90:0/2798294341 shutdown_connections 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815bd1d0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815b0b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2798294341 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815ba8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.778 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.778+0000 7f2312c7db80 1 -- 172.21.15.90:0/2798294341 >> 172.21.15.90:0/2798294341 conn(0x55b88150f6a0 msgr2=0x55b8815aba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.779+0000 7f2312c7db80 1 -- 172.21.15.90:0/2798294341 shutdown_connections 2024-04-06T14:18:17.779 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.779+0000 7f2312c7db80 1 -- 172.21.15.90:0/2798294341 wait complete. 2024-04-06T14:18:17.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.779+0000 7f2312c7db80 1 -- start start 2024-04-06T14:18:17.780 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.779+0000 7f2312c7db80 1 Processor -- start 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815b0780 0x55b8815b7bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815b8100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b88149dd80 con 0x55b8815b10a0 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b88149da20 con 0x55b8815b0780 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8813856a0 con 0x55b8815bade0 2024-04-06T14:18:17.782 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.781+0000 7f2312c7db80 10 client.0 _ll_register_callbacks cb 0x55b8814a7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:17.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44266/0 (socket says 172.21.15.90:44266) 2024-04-06T14:18:17.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 -- 172.21.15.90:0/1324208283 learned_addr learned my addr 172.21.15.90:0/1324208283 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.783 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f22fffff700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815b8100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2304865700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815b0780 0x55b8815b7bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 -- 172.21.15.90:0/1324208283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815b0780 msgr2=0x55b8815b7bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815b0780 0x55b8815b7bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 -- 172.21.15.90:0/1324208283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 msgr2=0x55b8815b8100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815b8100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f2305066700 1 -- 172.21.15.90:0/1324208283 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22f80099b0 con 0x55b8815bade0 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.782+0000 7f22fffff700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815b8100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f2305066700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f22f8012c10 tx=0x7f22f8012f50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e305c2c212574d15 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.784 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f22dffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f800e040 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f22f801b620 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b8814fa700 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f22f80127f0 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.783+0000 7f2312c7db80 10 client.27049 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b88132fb60 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22f802b030 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f22f801aa10 con 0x55b8815bade0 2024-04-06T14:18:17.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 10 client.27049 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:17.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 10 client.27049 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:18:17.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 20 client.27049 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:17.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b8815ca9c0 0x55b8815ccde0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 4 client.27049 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:17.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.784+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b881350cf0 con 0x55b8815ca9c0 2024-04-06T14:18:17.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.785+0000 7f2304865700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b8815ca9c0 0x55b8815ccde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.786+0000 7f2304865700 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b8815ca9c0 0x55b8815ccde0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=a308cf84a2bf3ca server_cookie=bd4ffc74a751a833 in_seq=0 out_seq=0 2024-04-06T14:18:17.787 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.786+0000 7f22dffff700 10 client.27049 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:17.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.787+0000 7f22dffff700 1 -- 172.21.15.90:0/1324208283 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b881350cf0 con 0x55b8815ca9c0 2024-04-06T14:18:17.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.787+0000 7f22dffff700 10 client.27049 handle_command_reply: tid=0 2024-04-06T14:18:17.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.787+0000 7f2312c7db80 1 client.27049 shutdown 2024-04-06T14:18:17.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 msgr2=0x55b8815b8640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f22f8012c10 tx=0x7f22f8012f50 comp rx=0 tx=0).stop 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 shutdown_connections 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b8815ca9c0 0x55b8815ccde0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815bade0 0x55b8815b8640 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815b0780 0x55b8815b7bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 --2- 172.21.15.90:0/1324208283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815b8100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 >> 172.21.15.90:0/1324208283 conn(0x55b88150f6a0 msgr2=0x55b8815aac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f22dffff700 0 client.27049 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 shutdown_connections 2024-04-06T14:18:17.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 1 -- 172.21.15.90:0/1324208283 wait complete. 2024-04-06T14:18:17.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.788+0000 7f2312c7db80 20 client.27049 trim_cache size 0 max 16384 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 Processor -- start 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 -- start start 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b10a0 0x55b8815b69d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815ab3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8815b7990 con 0x55b8815b0780 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8815ac7c0 con 0x55b8815bade0 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2312c7db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8815acef0 con 0x55b8815b10a0 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f22fffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b10a0 0x55b8815b69d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40478/0 (socket says 172.21.15.90:40478) 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 -- 172.21.15.90:0/3380013469 learned_addr learned my addr 172.21.15.90:0/3380013469 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2305066700 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815ab3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 -- 172.21.15.90:0/3380013469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b10a0 msgr2=0x55b8815b69d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b10a0 0x55b8815b69d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.805+0000 7f2304865700 1 -- 172.21.15.90:0/3380013469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 msgr2=0x55b8815ab3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f2304865700 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815ab3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f2304865700 1 -- 172.21.15.90:0/3380013469 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b8814c7ea0 con 0x55b8815b0780 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f2304865700 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f22f4097a80 tx=0x7f22f40a1040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad55f568977028b2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f22eb7fe700 1 -- 172.21.15.90:0/3380013469 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f4003df0 con 0x55b8815b0780 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f22eb7fe700 1 -- 172.21.15.90:0/3380013469 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22f409b710 con 0x55b8815b0780 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.806+0000 7f22eb7fe700 1 -- 172.21.15.90:0/3380013469 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f409b270 con 0x55b8815b0780 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 -- 172.21.15.90:0/3380013469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 msgr2=0x55b8815b6490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 secure :-1 s=READY pgs=811 cs=0 l=1 rev1=1 crypto rx=0x7f22f4097a80 tx=0x7f22f40a1040 comp rx=0 tx=0).stop 2024-04-06T14:18:17.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 -- 172.21.15.90:0/3380013469 shutdown_connections 2024-04-06T14:18:17.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815ab3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b0780 0x55b8815b6490 unknown :-1 s=CLOSED pgs=811 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 --2- 172.21.15.90:0/3380013469 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b10a0 0x55b8815b69d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.807+0000 7f2312c7db80 1 -- 172.21.15.90:0/3380013469 >> 172.21.15.90:0/3380013469 conn(0x55b88150f6a0 msgr2=0x55b8815b2f40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.808+0000 7f2312c7db80 1 -- 172.21.15.90:0/3380013469 shutdown_connections 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.808+0000 7f2312c7db80 1 -- 172.21.15.90:0/3380013469 wait complete. 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.808+0000 7f2312c7db80 1 -- start start 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.808+0000 7f2312c7db80 1 Processor -- start 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815bfa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815bff80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815c04c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b88161d8e0 con 0x55b8815b10a0 2024-04-06T14:18:17.812 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b88161da50 con 0x55b8815bade0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b88161dc10 con 0x55b8815b0780 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2312c7db80 10 client.0 _ll_register_callbacks cb 0x55b8814a7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2304865700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815bfa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2304865700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815bfa40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44286/0 (socket says 172.21.15.90:44286) 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2304865700 1 -- 172.21.15.90:0/2052217857 learned_addr learned my addr 172.21.15.90:0/2052217857 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f2305066700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815c04c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.809+0000 7f22fffff700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815bff80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 -- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 msgr2=0x55b8815bfa40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815bfa40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 -- 172.21.15.90:0/2052217857 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 msgr2=0x55b8815bff80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815bff80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 -- 172.21.15.90:0/2052217857 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f230007ef50 con 0x55b8815bade0 2024-04-06T14:18:17.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2305066700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815c04c0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f22f8012c40 tx=0x7f22f8026ce0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7bda4a550124c2db server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f22dffff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22f8034070 con 0x55b8815bade0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f22f80127f0 con 0x55b8815bade0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b8814c2380 con 0x55b8815bade0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.810+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f22f800e040 con 0x55b8815bade0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.811+0000 7f2312c7db80 10 client.27293 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.811+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b881387cd0 con 0x55b8815bade0 2024-04-06T14:18:17.814 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.811+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b8814c2380 con 0x55b8815bade0 2024-04-06T14:18:17.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f22f8037030 con 0x55b8815bade0 2024-04-06T14:18:17.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 10 client.27293 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:17.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 10 client.27293 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:18:17.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 20 client.27293 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:17.816 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b88161f0e0 0x55b881621500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 4 client.27293 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b881385ce0 con 0x55b88161f0e0 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2304865700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b88161f0e0 0x55b881621500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f22dffff700 10 client.27293 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.812+0000 7f2304865700 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b88161f0e0 0x55b881621500 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=a2a537bb8f49c2b9 server_cookie=5ff8f3e922c8670a in_seq=0 out_seq=0 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f22dffff700 1 -- 172.21.15.90:0/2052217857 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b881385ce0 con 0x55b88161f0e0 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f22dffff700 10 client.27293 handle_command_reply: tid=0 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 client.27293 shutdown 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 msgr2=0x55b8815c04c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815c04c0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f22f8012c40 tx=0x7f22f8026ce0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.817 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 shutdown_connections 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55b88161f0e0 0x55b881621500 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b8815bade0 0x55b8815c04c0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f22dffff700 0 client.27293 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b8815b0780 0x55b8815bfa40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 --2- 172.21.15.90:0/2052217857 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b8815b10a0 0x55b8815bff80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 >> 172.21.15.90:0/2052217857 conn(0x55b88150f6a0 msgr2=0x55b8815af770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 shutdown_connections 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 1 -- 172.21.15.90:0/2052217857 wait complete. 2024-04-06T14:18:17.818 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.813+0000 7f2312c7db80 20 client.27293 trim_cache size 0 max 16384 2024-04-06T14:18:17.819 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:17.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 msgr2=0x7f22f00a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 0x7f22f00a38e0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f22ec006fd0 tx=0x7f22ec008040 comp rx=0 tx=0).stop 2024-04-06T14:18:17.819 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 msgr2=0x7f2300081180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 secure :-1 s=READY pgs=810 cs=0 l=1 rev1=1 crypto rx=0x7f22f400c010 tx=0x7f22f40053a0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 shutdown_connections 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f22f00a1420 0x7f22f00a38e0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f230010d450 0x7f230007a2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2300107390 0x7f2300081180 unknown :-1 s=CLOSED pgs=810 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 --2- 172.21.15.90:0/1679069595 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2300107d50 0x7f2300081700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 >> 172.21.15.90:0/1679069595 conn(0x7f2300076ba0 msgr2=0x7f2300077d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 shutdown_connections 2024-04-06T14:18:17.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:17.814+0000 7f2306ac9700 1 -- 172.21.15.90:0/1679069595 wait complete. 2024-04-06T14:18:17.830 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:17.831 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:18:18.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 -- 172.21.15.90:0/3705129922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a01005a0 msgr2=0x7f71a0100980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 --2- 172.21.15.90:0/3705129922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a01005a0 0x7f71a0100980 secure :-1 s=READY pgs=812 cs=0 l=1 rev1=1 crypto rx=0x7f7190009a10 tx=0x7f7190009d20 comp rx=0 tx=0).stop 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 -- 172.21.15.90:0/3705129922 shutdown_connections 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 --2- 172.21.15.90:0/3705129922 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01051a0 0x7f71a010cd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 --2- 172.21.15.90:0/3705129922 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a01005a0 0x7f71a0100980 unknown :-1 s=CLOSED pgs=812 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 --2- 172.21.15.90:0/3705129922 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a0100f50 0x7f71a0104a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 -- 172.21.15.90:0/3705129922 >> 172.21.15.90:0/3705129922 conn(0x7f71a00761b0 msgr2=0x7f71a00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.004+0000 7f71a667f700 1 -- 172.21.15.90:0/3705129922 shutdown_connections 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.005+0000 7f71a667f700 1 -- 172.21.15.90:0/3705129922 wait complete. 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.005+0000 7f71a667f700 1 Processor -- start 2024-04-06T14:18:18.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.005+0000 7f71a667f700 1 -- start start 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 0x7f71a0195290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71a019a5f0 con 0x7f71a0199b70 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71a019a760 con 0x7f71a01005a0 2024-04-06T14:18:18.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a667f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f71a019a920 con 0x7f71a01051a0 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f719f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40502/0 (socket says 172.21.15.90:40502) 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f719f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44300/0 (socket says 172.21.15.90:44300) 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2620901495 learned_addr learned my addr 172.21.15.90:0/2620901495 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f719ffff700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 0x7f71a0195290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2620901495 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 msgr2=0x7f71a01957d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2620901495 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 msgr2=0x7f71a0195290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 0x7f71a0195290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2620901495 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71900095f0 con 0x7f71a0199b70 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f719f7fe700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.006+0000 7f719ffff700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 0x7f71a0195290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:18.009 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f719800b780 tx=0x7f719800ba90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f719d7fa700 1 -- 172.21.15.90:0/2620901495 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198011820 con 0x7f71a0199b70 2024-04-06T14:18:18.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f719d7fa700 1 -- 172.21.15.90:0/2620901495 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f719800d040 con 0x7f71a0199b70 2024-04-06T14:18:18.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f71a01a1e80 con 0x7f71a0199b70 2024-04-06T14:18:18.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f719d7fa700 1 -- 172.21.15.90:0/2620901495 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198010de0 con 0x7f71a0199b70 2024-04-06T14:18:18.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.007+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71a01a22b0 con 0x7f71a0199b70 2024-04-06T14:18:18.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.011+0000 7f719d7fa700 1 -- 172.21.15.90:0/2620901495 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f719801a400 con 0x7f71a0199b70 2024-04-06T14:18:18.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.012+0000 7f719d7fa700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 0x7f71840a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.012+0000 7f71b2833b80 1 Processor -- start 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.012+0000 7f719d7fa700 1 -- 172.21.15.90:0/2620901495 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f71980ccb20 con 0x7f71a0199b70 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.012+0000 7f719ffff700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 0x7f71840a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.012+0000 7f71b2833b80 1 -- start start 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad685db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad68678a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f719ffff700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 0x7f71840a3750 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f7190009960 tx=0x7f7190009f60 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad686a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad67b5990 con 0x555ad6867de0 2024-04-06T14:18:18.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad67b3350 con 0x555ad685e0a0 2024-04-06T14:18:18.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71b2833b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad67aec40 con 0x555ad685d780 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad686a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad686a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40506/0 (socket says 172.21.15.90:40506) 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f71a4c1c700 1 -- 172.21.15.90:0/340348340 learned_addr learned my addr 172.21.15.90:0/340348340 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f719ffff700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad685db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.013+0000 7f719f7fe700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad68678a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 -- 172.21.15.90:0/340348340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 msgr2=0x555ad68678a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad68678a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 -- 172.21.15.90:0/340348340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 msgr2=0x555ad686a1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad686a1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 -- 172.21.15.90:0/340348340 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555ad66303b0 con 0x555ad685d780 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719f7fe700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad68678a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f719ffff700 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad685db60 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f719000b540 tx=0x7f7190022000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4bc298ebfa824c87 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f71967fc700 1 -- 172.21.15.90:0/340348340 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71900051c0 con 0x555ad685d780 2024-04-06T14:18:18.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f71967fc700 1 -- 172.21.15.90:0/340348340 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f719000b6f0 con 0x555ad685d780 2024-04-06T14:18:18.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.014+0000 7f71967fc700 1 -- 172.21.15.90:0/340348340 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7190004760 con 0x555ad685d780 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 -- 172.21.15.90:0/340348340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 msgr2=0x555ad685db60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad685db60 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f719000b540 tx=0x7f7190022000 comp rx=0 tx=0).stop 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 -- 172.21.15.90:0/340348340 shutdown_connections 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad686a1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad685db60 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 --2- 172.21.15.90:0/340348340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad68678a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 -- 172.21.15.90:0/340348340 >> 172.21.15.90:0/340348340 conn(0x555ad67bc6a0 msgr2=0x555ad6858a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 -- 172.21.15.90:0/340348340 shutdown_connections 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.015+0000 7f71b2833b80 1 -- 172.21.15.90:0/340348340 wait complete. 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 -- start start 2024-04-06T14:18:18.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 Processor -- start 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad68677e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad6864b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad674ad80 con 0x555ad6867de0 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad674aa20 con 0x555ad685e0a0 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad66326a0 con 0x555ad685d780 2024-04-06T14:18:18.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.016+0000 7f71b2833b80 10 client.0 _ll_register_callbacks cb 0x555ad6754470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40512/0 (socket says 172.21.15.90:40512) 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2849389660 learned_addr learned my addr 172.21.15.90:0/2849389660 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f719ffff700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad68677e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f719f7fe700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad6864b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2849389660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 msgr2=0x555ad68677e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad68677e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 msgr2=0x555ad6864b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad6864b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 -- 172.21.15.90:0/2849389660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f718c007c50 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f719800bde0 tx=0x7f7198099ac0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c62ba4909dd6df08 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.017+0000 7f71837fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7198099be0 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f719809da10 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555ad676f380 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f71980a2060 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 10 client.17580 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f71a010d590 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7198063020 con 0x555ad6867de0 2024-04-06T14:18:18.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f71980a1c60 con 0x555ad6867de0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 10 client.17580 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 10 client.17580 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 20 client.17580 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68779f0 0x555ad6879e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 4 client.17580 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555ad67a7700 con 0x555ad68779f0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.018+0000 7f719ffff700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68779f0 0x555ad6879e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.019+0000 7f71837fe700 10 client.17580 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.019+0000 7f719ffff700 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68779f0 0x555ad6879e10 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f6c868ec63a32923 server_cookie=9b76dbb4c2cb8c55 in_seq=0 out_seq=0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.019+0000 7f71837fe700 1 -- 172.21.15.90:0/2849389660 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555ad67a7700 con 0x555ad68779f0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.019+0000 7f71837fe700 10 client.17580 handle_command_reply: tid=0 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.019+0000 7f71b2833b80 1 client.17580 shutdown 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 msgr2=0x555ad68651f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 secure :-1 s=READY pgs=814 cs=0 l=1 rev1=1 crypto rx=0x7f719800bde0 tx=0x7f7198099ac0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 shutdown_connections 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68651f0 unknown :-1 s=CLOSED pgs=814 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68779f0 0x555ad6879e10 crc :-1 s=READY pgs=219 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685d780 0x555ad68677e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 --2- 172.21.15.90:0/2849389660 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad6864b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 >> 172.21.15.90:0/2849389660 conn(0x555ad67bc6a0 msgr2=0x555ad6857c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71837fe700 0 client.17580 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 shutdown_connections 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 1 -- 172.21.15.90:0/2849389660 wait complete. 2024-04-06T14:18:18.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.020+0000 7f71b2833b80 20 client.17580 trim_cache size 0 max 16384 2024-04-06T14:18:18.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 750.0 s, 1598.9 tps, lat 63.133 ms stddev 70.872 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.040+0000 7f71b2833b80 1 Processor -- start 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.040+0000 7f71b2833b80 1 -- start start 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.040+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad685d780 0x555ad6865170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad686ea00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71b2833b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad687a1a0 con 0x555ad685d780 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71b2833b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad686ef40 con 0x555ad685e0a0 2024-04-06T14:18:18.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71b2833b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad6863030 con 0x555ad6867de0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f719ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad685d780 0x555ad6865170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f719f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad686ea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44324/0 (socket says 172.21.15.90:44324) 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 -- 172.21.15.90:0/1700458390 learned_addr learned my addr 172.21.15.90:0/1700458390 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 -- 172.21.15.90:0/1700458390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 msgr2=0x555ad686ea00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad686ea00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 -- 172.21.15.90:0/1700458390 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad685d780 msgr2=0x555ad6865170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad685d780 0x555ad6865170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 -- 172.21.15.90:0/1700458390 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555ad65dcb60 con 0x555ad6867de0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.041+0000 7f71a4c1c700 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f71980ca2d0 tx=0x7f7198010840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e159f9f02eebc8aa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.042+0000 7f7196ffd700 1 -- 172.21.15.90:0/1700458390 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71980b23f0 con 0x555ad6867de0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.042+0000 7f7196ffd700 1 -- 172.21.15.90:0/1700458390 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7198063070 con 0x555ad6867de0 2024-04-06T14:18:18.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.042+0000 7f7196ffd700 1 -- 172.21.15.90:0/1700458390 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f71980a2060 con 0x555ad6867de0 2024-04-06T14:18:18.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 -- 172.21.15.90:0/1700458390 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 msgr2=0x555ad6863440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f71980ca2d0 tx=0x7f7198010840 comp rx=0 tx=0).stop 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 -- 172.21.15.90:0/1700458390 shutdown_connections 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad6867de0 0x555ad6863440 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad685d780 0x555ad6865170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 --2- 172.21.15.90:0/1700458390 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685e0a0 0x555ad686ea00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 -- 172.21.15.90:0/1700458390 >> 172.21.15.90:0/1700458390 conn(0x555ad67bc6a0 msgr2=0x555ad687e730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 -- 172.21.15.90:0/1700458390 shutdown_connections 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.043+0000 7f71b2833b80 1 -- 172.21.15.90:0/1700458390 wait complete. 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.044+0000 7f71b2833b80 1 -- start start 2024-04-06T14:18:18.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.044+0000 7f71b2833b80 1 Processor -- start 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685e0a0 0x555ad686cc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68cebf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad68cf130 con 0x555ad6867de0 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad68cf270 con 0x555ad685d780 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ad68cf3b0 con 0x555ad685e0a0 2024-04-06T14:18:18.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71b2833b80 10 client.0 _ll_register_callbacks cb 0x555ad6754470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:18.049 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:18.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68cebf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59136/0 (socket says 172.21.15.90:59136) 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f71a4c1c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68cebf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40538/0 (socket says 172.21.15.90:40538) 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 -- 172.21.15.90:0/164761309 learned_addr learned my addr 172.21.15.90:0/164761309 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719f7fe700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685e0a0 0x555ad686cc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 -- 172.21.15.90:0/164761309 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685e0a0 msgr2=0x555ad686cc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685e0a0 0x555ad686cc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 -- 172.21.15.90:0/164761309 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 msgr2=0x555ad68cebf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68cebf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.045+0000 7f719ffff700 1 -- 172.21.15.90:0/164761309 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f71a010d590 con 0x555ad685d780 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f719ffff700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f7190004060 tx=0x7f7190005da0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=adf5d09b12d05d6f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f719003f3b0 con 0x555ad685d780 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f719003f510 con 0x555ad685d780 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f719000b6f0 con 0x555ad685d780 2024-04-06T14:18:18.050 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f719000b860 con 0x555ad685d780 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 10 client.27308 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555ad6774ea0 con 0x555ad685d780 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f719000b6f0 con 0x555ad685d780 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7190004500 con 0x555ad685d780 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 10 client.27308 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 10 client.27308 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 20 client.27308 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68d0890 0x555ad68d2cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 4 client.27308 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.046+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x555ad65fdcf0 con 0x555ad68d0890 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f719f7fe700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68d0890 0x555ad68d2cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71837fe700 10 client.27308 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f719f7fe700 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68d0890 0x555ad68d2cb0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9fe352700ae7d592 server_cookie=59bbf00a1c2a252e in_seq=0 out_seq=0 2024-04-06T14:18:18.051 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71837fe700 1 -- 172.21.15.90:0/164761309 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x555ad65fdcf0 con 0x555ad68d0890 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71837fe700 10 client.27308 handle_command_reply: tid=0 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 client.27308 shutdown 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 msgr2=0x555ad686c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f7190004060 tx=0x7f7190005da0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 shutdown_connections 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ad6867de0 0x555ad68cebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ad685d780 0x555ad686c730 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.047+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555ad68d0890 0x555ad68d2cb0 crc :-1 s=READY pgs=220 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71b2833b80 1 --2- 172.21.15.90:0/164761309 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ad685e0a0 0x555ad686cc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71837fe700 0 client.27308 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 >> 172.21.15.90:0/164761309 conn(0x555ad67bc6a0 msgr2=0x555ad68633c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 shutdown_connections 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71b2833b80 1 -- 172.21.15.90:0/164761309 wait complete. 2024-04-06T14:18:18.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71b2833b80 20 client.27308 trim_cache size 0 max 16384 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 msgr2=0x7f71840a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 0x7f71840a3750 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f7190009960 tx=0x7f7190009f60 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 msgr2=0x7f71a0199ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 secure :-1 s=READY pgs=813 cs=0 l=1 rev1=1 crypto rx=0x7f719800b780 tx=0x7f719800ba90 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 shutdown_connections 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f71840a1290 0x7f71840a3750 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f71a0199b70 0x7f71a0199ff0 unknown :-1 s=CLOSED pgs=813 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f71a01005a0 0x7f71a0195290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 --2- 172.21.15.90:0/2620901495 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f71a01051a0 0x7f71a01957d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 >> 172.21.15.90:0/2620901495 conn(0x7f71a00761b0 msgr2=0x7f71a01034a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 shutdown_connections 2024-04-06T14:18:18.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:18.048+0000 7f71a667f700 1 -- 172.21.15.90:0/2620901495 wait complete. 2024-04-06T14:18:19.067 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:18:19.067 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:18:19.067 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:18:19.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:18 smithi105 ceph-mon[46969]: pgmap v403: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 607 op/s 2024-04-06T14:18:19.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:18 smithi090 ceph-mon[43420]: pgmap v403: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 607 op/s 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 -- 172.21.15.90:0/1871602321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 msgr2=0x7f727c0ff6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 --2- 172.21.15.90:0/1871602321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c0ff6a0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f7268009990 tx=0x7f7268009ca0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 -- 172.21.15.90:0/1871602321 shutdown_connections 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 --2- 172.21.15.90:0/1871602321 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c0ffbe0 0x7f727c10dbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 --2- 172.21.15.90:0/1871602321 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c105d00 0x7f727c1060e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 --2- 172.21.15.90:0/1871602321 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c0ff6a0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 -- 172.21.15.90:0/1871602321 >> 172.21.15.90:0/1871602321 conn(0x7f727c0761b0 msgr2=0x7f727c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 -- 172.21.15.90:0/1871602321 shutdown_connections 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.361+0000 7f7282d37700 1 -- 172.21.15.90:0/1871602321 wait complete. 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 Processor -- start 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 -- start start 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c10db60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c0ffbe0 0x7f727c10ac30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f727c1adfe0 con 0x7f727c0ffbe0 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f727c1ae120 con 0x7f727c105d00 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7282d37700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f727c1ae260 con 0x7f727c0ff220 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f72812d4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f72812d4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59158/0 (socket says 172.21.15.90:59158) 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f72812d4700 1 -- 172.21.15.90:0/1644331607 learned_addr learned my addr 172.21.15.90:0/1644331607 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f727bfff700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c0ffbe0 0x7f727c10ac30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.362+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c10db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 -- 172.21.15.90:0/1644331607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 msgr2=0x7f727c10db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c10db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 -- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c0ffbe0 msgr2=0x7f727c10ac30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c0ffbe0 0x7f727c10ac30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 -- 172.21.15.90:0/1644331607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72680095f0 con 0x7f727c105d00 2024-04-06T14:18:19.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c10db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f72812d4700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f727400b7b0 tx=0x7f727400bac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7279ffb700 1 -- 172.21.15.90:0/1644331607 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7274011800 con 0x7f727c105d00 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7279ffb700 1 -- 172.21.15.90:0/1644331607 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7274011e20 con 0x7f727c105d00 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7279ffb700 1 -- 172.21.15.90:0/1644331607 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7274007e80 con 0x7f727c105d00 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f727c1ae4c0 con 0x7f727c105d00 2024-04-06T14:18:19.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.363+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f727c06ceb0 con 0x7f727c105d00 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.368+0000 7f728eeebb80 1 Processor -- start 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.368+0000 7f728eeebb80 1 -- start start 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f8a920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f8ae60 0x559ab0f8d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0ed8a10 con 0x559ab0f81120 2024-04-06T14:18:19.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0ed63d0 con 0x559ab0f80800 2024-04-06T14:18:19.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.369+0000 7f728eeebb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0ed1cc0 con 0x559ab0f8ae60 2024-04-06T14:18:19.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f7279ffb700 1 -- 172.21.15.90:0/1644331607 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f72740074e0 con 0x7f727c105d00 2024-04-06T14:18:19.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59174/0 (socket says 172.21.15.90:59174) 2024-04-06T14:18:19.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 -- 172.21.15.90:0/377242800 learned_addr learned my addr 172.21.15.90:0/377242800 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.371 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f8ae60 0x559ab0f8d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f7280ad3700 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f8a920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 -- 172.21.15.90:0/377242800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f8ae60 msgr2=0x559ab0f8d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f8ae60 0x559ab0f8d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 -- 172.21.15.90:0/377242800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 msgr2=0x559ab0f8a920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f8a920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f727bfff700 1 -- 172.21.15.90:0/377242800 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559ab0d53430 con 0x559ab0f80800 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.370+0000 7f7279ffb700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 0x7f726c0a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f727bfff700 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f7268009990 tx=0x7f726801a040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ac66fe1109cae1f2 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 0x7f726c0a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f7279ffb700 1 -- 172.21.15.90:0/1644331607 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f727400e070 con 0x7f727c105d00 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f7266ffd700 1 -- 172.21.15.90:0/377242800 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f726800bb30 con 0x559ab0f80800 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f7266ffd700 1 -- 172.21.15.90:0/377242800 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f726800bc90 con 0x559ab0f80800 2024-04-06T14:18:19.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.371+0000 7f7266ffd700 1 -- 172.21.15.90:0/377242800 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7268017650 con 0x559ab0f80800 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 -- 172.21.15.90:0/377242800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 msgr2=0x559ab0f80be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 0x7f726c0a3920 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f7270012640 tx=0x7f7270009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7f7268009990 tx=0x7f726801a040 comp rx=0 tx=0).stop 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 -- 172.21.15.90:0/377242800 shutdown_connections 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f8ae60 0x559ab0f8d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f80be0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 --2- 172.21.15.90:0/377242800 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f8a920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.372+0000 7f728eeebb80 1 -- 172.21.15.90:0/377242800 >> 172.21.15.90:0/377242800 conn(0x559ab0edf720 msgr2=0x559ab0f7bb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- 172.21.15.90:0/377242800 shutdown_connections 2024-04-06T14:18:19.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- 172.21.15.90:0/377242800 wait complete. 2024-04-06T14:18:19.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- start start 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 Processor -- start 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f87ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0f882e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0e6de00 con 0x559ab0f80800 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0e6daa0 con 0x559ab0f8ae60 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0d55720 con 0x559ab0f81120 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.373+0000 7f728eeebb80 10 client.0 _ll_register_callbacks cb 0x559ab0e774f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40586/0 (socket says 172.21.15.90:40586) 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 -- 172.21.15.90:0/1840101935 learned_addr learned my addr 172.21.15.90:0/1840101935 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f727bfff700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f87ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f72812d4700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0f882e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 -- 172.21.15.90:0/1840101935 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 msgr2=0x559ab0f87ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f87ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 -- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 msgr2=0x559ab0f882e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0f882e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 -- 172.21.15.90:0/1840101935 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7268020040 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f7280ad3700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f727001ea60 tx=0x7f72700047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bd9059e129bb2f52 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.374+0000 7f725bfff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7270023070 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f727001ed70 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559ab0e92400 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7270017070 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 10 client.17598 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559ab0cffbe0 con 0x559ab0f80800 2024-04-06T14:18:19.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559ab0e92400 con 0x559ab0f80800 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f727002a630 con 0x559ab0f80800 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 10 client.17598 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 10 client.17598 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 20 client.17598 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0f9ab00 0x559ab0f9cf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 4 client.17598 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.375+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559ab0eca780 con 0x559ab0f9ab00 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f727bfff700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0f9ab00 0x559ab0f9cf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f727bfff700 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0f9ab00 0x559ab0f9cf20 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=78f186418d8617bc server_cookie=2254187f74d8b838 in_seq=0 out_seq=0 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f725bfff700 10 client.17598 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f725bfff700 1 -- 172.21.15.90:0/1840101935 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x559ab0eca780 con 0x559ab0f9ab00 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f725bfff700 10 client.17598 handle_command_reply: tid=0 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.376+0000 7f728eeebb80 1 client.17598 shutdown 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 msgr2=0x559ab0f8a8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 secure :-1 s=READY pgs=815 cs=0 l=1 rev1=1 crypto rx=0x7f727001ea60 tx=0x7f72700047c0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 shutdown_connections 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0f882e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f80800 0x559ab0f8a8b0 unknown :-1 s=CLOSED pgs=815 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0f9ab00 0x559ab0f9cf20 crc :-1 s=READY pgs=221 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 --2- 172.21.15.90:0/1840101935 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f87ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f725bfff700 0 client.17598 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 >> 172.21.15.90:0/1840101935 conn(0x559ab0edf720 msgr2=0x559ab0f7aca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 shutdown_connections 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 1 -- 172.21.15.90:0/1840101935 wait complete. 2024-04-06T14:18:19.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.377+0000 7f728eeebb80 20 client.17598 trim_cache size 0 max 16384 2024-04-06T14:18:19.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:18 smithi134 ceph-mon[47549]: pgmap v403: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 607 op/s 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 Processor -- start 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 -- start start 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f7ff00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 0x559ab0f8a3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0f7cf90 con 0x559ab0f8ae60 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.399+0000 7f728eeebb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0f88640 con 0x559ab0f80800 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f728eeebb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0f91fc0 con 0x559ab0f81120 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59196/0 (socket says 172.21.15.90:59196) 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 -- 172.21.15.90:0/4152690401 learned_addr learned my addr 172.21.15.90:0/4152690401 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f72812d4700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 0x559ab0f8a3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f727bfff700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f7ff00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 -- 172.21.15.90:0/4152690401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 msgr2=0x559ab0f7ff00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f7ff00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 -- 172.21.15.90:0/4152690401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 msgr2=0x559ab0f8a3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 0x559ab0f8a3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 -- 172.21.15.90:0/4152690401 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559ab0fa1b10 con 0x559ab0f80800 2024-04-06T14:18:19.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f72812d4700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 0x559ab0f8a3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:19.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f7280ad3700 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f7270012e10 tx=0x7f7270005dc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=243664bbcd45b924 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f72677fe700 1 -- 172.21.15.90:0/4152690401 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7270023040 con 0x559ab0f80800 2024-04-06T14:18:19.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f72677fe700 1 -- 172.21.15.90:0/4152690401 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f727001ed70 con 0x559ab0f80800 2024-04-06T14:18:19.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.400+0000 7f72677fe700 1 -- 172.21.15.90:0/4152690401 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f727001d410 con 0x559ab0f80800 2024-04-06T14:18:19.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 -- 172.21.15.90:0/4152690401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 msgr2=0x559ab0f91a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7f7270012e10 tx=0x7f7270005dc0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 -- 172.21.15.90:0/4152690401 shutdown_connections 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f8ae60 0x559ab0f8a3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f80800 0x559ab0f91a80 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 --2- 172.21.15.90:0/4152690401 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f81120 0x559ab0f7ff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 -- 172.21.15.90:0/4152690401 >> 172.21.15.90:0/4152690401 conn(0x559ab0edf720 msgr2=0x559ab0f7f7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 -- 172.21.15.90:0/4152690401 shutdown_connections 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.401+0000 7f728eeebb80 1 -- 172.21.15.90:0/4152690401 wait complete. 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 -- start start 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 Processor -- start 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f80800 0x559ab0f907c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.407 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0feee40 con 0x559ab0f81120 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0feef80 con 0x559ab0f8ae60 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x559ab0fef140 con 0x559ab0f80800 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f728eeebb80 10 client.0 _ll_register_callbacks cb 0x559ab0e774f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f727bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f72812d4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f727bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40596/0 (socket says 172.21.15.90:40596) 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f727bfff700 1 -- 172.21.15.90:0/2436512355 learned_addr learned my addr 172.21.15.90:0/2436512355 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f72812d4700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59204/0 (socket says 172.21.15.90:59204) 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.402+0000 7f7280ad3700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f80800 0x559ab0f907c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 -- 172.21.15.90:0/2436512355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f80800 msgr2=0x559ab0f907c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f80800 0x559ab0f907c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 -- 172.21.15.90:0/2436512355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 msgr2=0x559ab0f90d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 -- 172.21.15.90:0/2436512355 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f727c06ceb0 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f727bfff700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f72812d4700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f72740ca1a0 tx=0x7f72740995d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9228008fb7dc3b17 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72740a1fe0 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f726800bc90 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f72740b1070 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f72740b2290 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 10 client.27323 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x559ab0cffbe0 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f726800bc90 con 0x559ab0f8ae60 2024-04-06T14:18:19.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f72740a1b50 con 0x559ab0f8ae60 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 10 client.27323 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 10 client.27323 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 20 client.27323 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0ff09e0 0x559ab0ff2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 4 client.27323 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.403+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x559ab0d20d70 con 0x559ab0ff09e0 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.404+0000 7f7280ad3700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0ff09e0 0x559ab0ff2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.404+0000 7f725bfff700 10 client.27323 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.404+0000 7f7280ad3700 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0ff09e0 0x559ab0ff2e00 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cbe64bfa497645e7 server_cookie=b72f19710da4f6c1 in_seq=0 out_seq=0 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f725bfff700 1 -- 172.21.15.90:0/2436512355 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x559ab0d20d70 con 0x559ab0ff09e0 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f725bfff700 10 client.27323 handle_command_reply: tid=0 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 client.27323 shutdown 2024-04-06T14:18:19.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 msgr2=0x559ab0fee900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f72740ca1a0 tx=0x7f72740995d0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 shutdown_connections 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x559ab0f8ae60 0x559ab0fee900 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x559ab0ff09e0 0x559ab0ff2e00 crc :-1 s=READY pgs=222 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x559ab0f80800 0x559ab0f907c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f725bfff700 0 client.27323 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 --2- 172.21.15.90:0/2436512355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x559ab0f81120 0x559ab0f90d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 >> 172.21.15.90:0/2436512355 conn(0x559ab0edf720 msgr2=0x559ab0f7f7f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.405+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 shutdown_connections 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.406+0000 7f728eeebb80 1 -- 172.21.15.90:0/2436512355 wait complete. 2024-04-06T14:18:19.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.406+0000 7f728eeebb80 20 client.27323 trim_cache size 0 max 16384 2024-04-06T14:18:19.412 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:19.412 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:18:19.412 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "7322034a-02d1-43d9-83d1-849396ab9680", 2024-04-06T14:18:19.412 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:18:19.412 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 msgr2=0x7f726c0a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 0x7f726c0a3920 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f7270012640 tx=0x7f7270009260 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 msgr2=0x7f727c10b350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7f727400b7b0 tx=0x7f727400bac0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 shutdown_connections 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f726c0a1460 0x7f726c0a3920 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f727c105d00 0x7f727c10b350 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f727c0ff220 0x7f727c10db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 --2- 172.21.15.90:0/1644331607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f727c0ffbe0 0x7f727c10ac30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.407+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 >> 172.21.15.90:0/1644331607 conn(0x7f727c0761b0 msgr2=0x7f727c072840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.410+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 shutdown_connections 2024-04-06T14:18:19.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.410+0000 7f7282d37700 1 -- 172.21.15.90:0/1644331607 wait complete. 2024-04-06T14:18:19.428 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:18:19.429 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 -- 172.21.15.90:0/2083909598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46200683a0 msgr2=0x7f4620068820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 --2- 172.21.15.90:0/2083909598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46200683a0 0x7f4620068820 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7f460c0098f0 tx=0x7f460c009c00 comp rx=0 tx=0).stop 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 -- 172.21.15.90:0/2083909598 shutdown_connections 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 --2- 172.21.15.90:0/2083909598 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620103740 0x7f4620103bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 --2- 172.21.15.90:0/2083909598 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4620105c40 0x7f4620067e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.631 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 --2- 172.21.15.90:0/2083909598 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f46200683a0 0x7f4620068820 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 -- 172.21.15.90:0/2083909598 >> 172.21.15.90:0/2083909598 conn(0x7f4620076d50 msgr2=0x7f4620077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 -- 172.21.15.90:0/2083909598 shutdown_connections 2024-04-06T14:18:19.632 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.630+0000 7f4626fd9700 1 -- 172.21.15.90:0/2083909598 wait complete. 2024-04-06T14:18:19.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 Processor -- start 2024-04-06T14:18:19.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 -- start start 2024-04-06T14:18:19.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46200683a0 0x7f46201a3470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.635 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f462019dcd0 con 0x7f4620105c40 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f462019de40 con 0x7f4620103740 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4626fd9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f462019e000 con 0x7f46200683a0 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4625576700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f461ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4624d75700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46200683a0 0x7f46201a3470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4625576700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40616/0 (socket says 172.21.15.90:40616) 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f461ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59222/0 (socket says 172.21.15.90:59222) 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f4625576700 1 -- 172.21.15.90:0/4198966637 learned_addr learned my addr 172.21.15.90:0/4198966637 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f461ffff700 1 -- 172.21.15.90:0/4198966637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46200683a0 msgr2=0x7f46201a3470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.631+0000 7f461ffff700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46200683a0 0x7f46201a3470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461ffff700 1 -- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 msgr2=0x7f462019d670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461ffff700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461ffff700 1 -- 172.21.15.90:0/4198966637 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f460c009520 con 0x7f4620103740 2024-04-06T14:18:19.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f4625576700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:19.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461ffff700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f460c009fd0 tx=0x7f460c00f9e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461dffb700 1 -- 172.21.15.90:0/4198966637 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f460c01c070 con 0x7f4620103740 2024-04-06T14:18:19.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461dffb700 1 -- 172.21.15.90:0/4198966637 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f460c00baa0 con 0x7f4620103740 2024-04-06T14:18:19.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f461dffb700 1 -- 172.21.15.90:0/4198966637 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f460c0038e0 con 0x7f4620103740 2024-04-06T14:18:19.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f462019e280 con 0x7f4620103740 2024-04-06T14:18:19.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.632+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46201aa510 con 0x7f4620103740 2024-04-06T14:18:19.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.637+0000 7f463318db80 1 Processor -- start 2024-04-06T14:18:19.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.637+0000 7f463318db80 1 -- start start 2024-04-06T14:18:19.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fac0a0 0x555826fb58a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fb5de0 0x555826fb81d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826f03990 con 0x555826fb5de0 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826f01350 con 0x555826fab780 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f463318db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826efcc40 con 0x555826fac0a0 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59234/0 (socket says 172.21.15.90:59234) 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 -- 172.21.15.90:0/24958500 learned_addr learned my addr 172.21.15.90:0/24958500 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4624d75700 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fac0a0 0x555826fb58a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 -- 172.21.15.90:0/24958500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fac0a0 msgr2=0x555826fb58a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fac0a0 0x555826fb58a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 -- 172.21.15.90:0/24958500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fb5de0 msgr2=0x555826fb81d0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:18:19.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fb5de0 0x555826fb81d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.638+0000 7f4625576700 1 -- 172.21.15.90:0/24958500 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555826d7e3b0 con 0x555826fab780 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.639+0000 7f461dffb700 1 -- 172.21.15.90:0/4198966637 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f460c020030 con 0x7f4620103740 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.639+0000 7f4625576700 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f4618009570 tx=0x7f46180099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9bfbf7368658c4cc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.640+0000 7f460affd700 1 -- 172.21.15.90:0/24958500 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f461800e020 con 0x555826fab780 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.640+0000 7f460affd700 1 -- 172.21.15.90:0/24958500 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f461800f040 con 0x555826fab780 2024-04-06T14:18:19.644 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.640+0000 7f460affd700 1 -- 172.21.15.90:0/24958500 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4618028a80 con 0x555826fab780 2024-04-06T14:18:19.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.640+0000 7f461dffb700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 0x7f46100a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.645 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.640+0000 7f461dffb700 1 -- 172.21.15.90:0/4198966637 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f460c0cdae0 con 0x7f4620103740 2024-04-06T14:18:19.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.641+0000 7f463318db80 1 -- 172.21.15.90:0/24958500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 msgr2=0x555826fabb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.641+0000 7f463318db80 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f4618009570 tx=0x7f46180099e0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.642+0000 7f463318db80 1 -- 172.21.15.90:0/24958500 shutdown_connections 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.642+0000 7f463318db80 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fb5de0 0x555826fb81d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.642+0000 7f463318db80 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fabb60 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.642+0000 7f463318db80 1 --2- 172.21.15.90:0/24958500 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fac0a0 0x555826fb58a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.642+0000 7f463318db80 1 -- 172.21.15.90:0/24958500 >> 172.21.15.90:0/24958500 conn(0x555826f0a6a0 msgr2=0x555826fa6a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.643+0000 7f4624d75700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 0x7f46100a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.643+0000 7f463318db80 1 -- 172.21.15.90:0/24958500 shutdown_connections 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.643+0000 7f463318db80 1 -- 172.21.15.90:0/24958500 wait complete. 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.644+0000 7f463318db80 1 -- start start 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.644+0000 7f4624d75700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 0x7f46100a3750 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4614005950 tx=0x7f461400b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.644+0000 7f463318db80 1 Processor -- start 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fb2ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fac0a0 0x555826fb30e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826e98d80 con 0x555826fac0a0 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826e98a20 con 0x555826fab780 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826d806a0 con 0x555826fb5de0 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f463318db80 10 client.0 _ll_register_callbacks cb 0x555826ea2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:19.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:44442/0 (socket says 172.21.15.90:44442) 2024-04-06T14:18:19.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4624d75700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fb2ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 -- 172.21.15.90:0/3354765431 learned_addr learned my addr 172.21.15.90:0/3354765431 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f461ffff700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fac0a0 0x555826fb30e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 -- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 msgr2=0x555826fb2ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fb2ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 -- 172.21.15.90:0/3354765431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fac0a0 msgr2=0x555826fb30e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fac0a0 0x555826fb30e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 -- 172.21.15.90:0/3354765431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4618009220 con 0x555826fb5de0 2024-04-06T14:18:19.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4624d75700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fb2ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f4625576700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f4618009de0 tx=0x7f4618006240 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b2e401cbe2d6ada8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.645+0000 7f45fffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4618003a00 con 0x555826fb5de0 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f4618003b60 con 0x555826fb5de0 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555826ebd380 con 0x555826fb5de0 2024-04-06T14:18:19.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4618003d20 con 0x555826fb5de0 2024-04-06T14:18:19.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 10 client.27103 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:19.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555826d2ab60 con 0x555826fb5de0 2024-04-06T14:18:19.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555826ebd380 con 0x555826fb5de0 2024-04-06T14:18:19.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4618024040 con 0x555826fb5de0 2024-04-06T14:18:19.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 10 client.27103 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 10 client.27103 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 20 client.27103 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555826fc58c0 0x555826fc7ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 4 client.27103 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.646+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555826ef5700 con 0x555826fc58c0 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.647+0000 7f4624d75700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555826fc58c0 0x555826fc7ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.647+0000 7f4624d75700 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555826fc58c0 0x555826fc7ce0 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=32b2fac93e12520d server_cookie=49145ff78d00b59a in_seq=0 out_seq=0 2024-04-06T14:18:19.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.647+0000 7f45fffff700 10 client.27103 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f45fffff700 1 -- 172.21.15.90:0/3354765431 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555826ef5700 con 0x555826fc58c0 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f45fffff700 10 client.27103 handle_command_reply: tid=0 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 client.27103 shutdown 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 msgr2=0x555826fb3620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f4618009de0 tx=0x7f4618006240 comp rx=0 tx=0).stop 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 shutdown_connections 2024-04-06T14:18:19.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555826fc58c0 0x555826fc7ce0 crc :-1 s=READY pgs=223 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb3620 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fab780 0x555826fb2ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 --2- 172.21.15.90:0/3354765431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fac0a0 0x555826fb30e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.648+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 >> 172.21.15.90:0/3354765431 conn(0x555826f0a6a0 msgr2=0x555826fa5c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.649+0000 7f45fffff700 0 client.27103 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.651+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 shutdown_connections 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.651+0000 7f463318db80 1 -- 172.21.15.90:0/3354765431 wait complete. 2024-04-06T14:18:19.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.651+0000 7f463318db80 20 client.27103 trim_cache size 0 max 16384 2024-04-06T14:18:19.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 Processor -- start 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 -- start start 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fa6c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fafe20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb0360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826fac4b0 con 0x555826fab780 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826facfe0 con 0x555826fac0a0 2024-04-06T14:18:19.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.672+0000 7f463318db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826fae7b0 con 0x555826fb5de0 2024-04-06T14:18:19.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb0360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f461ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fafe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f461ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fafe20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:59258/0 (socket says 172.21.15.90:59258) 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f461ffff700 1 -- 172.21.15.90:0/766851868 learned_addr learned my addr 172.21.15.90:0/766851868 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4624d75700 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fa6c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 -- 172.21.15.90:0/766851868 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 msgr2=0x555826fafe20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fafe20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 -- 172.21.15.90:0/766851868 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 msgr2=0x555826fa6c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fa6c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.673+0000 7f4625576700 1 -- 172.21.15.90:0/766851868 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555826ec2ea0 con 0x555826fb5de0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.674+0000 7f4625576700 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb0360 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f4618031a40 tx=0x7f4618008f50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5db7ff56bedabae1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.674+0000 7f460b7fe700 1 -- 172.21.15.90:0/766851868 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4618007c30 con 0x555826fb5de0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.674+0000 7f460b7fe700 1 -- 172.21.15.90:0/766851868 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f46180284f0 con 0x555826fb5de0 2024-04-06T14:18:19.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.674+0000 7f460b7fe700 1 -- 172.21.15.90:0/766851868 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4618003b80 con 0x555826fb5de0 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- 172.21.15.90:0/766851868 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 msgr2=0x555826fb0360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb0360 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f4618031a40 tx=0x7f4618008f50 comp rx=0 tx=0).stop 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- 172.21.15.90:0/766851868 shutdown_connections 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fb0360 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fa6c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 --2- 172.21.15.90:0/766851868 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fafe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- 172.21.15.90:0/766851868 >> 172.21.15.90:0/766851868 conn(0x555826f0a6a0 msgr2=0x555826fa6520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- 172.21.15.90:0/766851868 shutdown_connections 2024-04-06T14:18:19.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- 172.21.15.90:0/766851868 wait complete. 2024-04-06T14:18:19.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 -- start start 2024-04-06T14:18:19.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.675+0000 7f463318db80 1 Processor -- start 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fbc9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fbab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826fbb130 con 0x555826fab780 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826fbb2a0 con 0x555826fac0a0 2024-04-06T14:18:19.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555826fbb460 con 0x555826fb5de0 2024-04-06T14:18:19.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f463318db80 10 client.0 _ll_register_callbacks cb 0x555826ea2470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:19.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f4624d75700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f4624d75700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:40640/0 (socket says 172.21.15.90:40640) 2024-04-06T14:18:19.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f4624d75700 1 -- 172.21.15.90:0/2614622726 learned_addr learned my addr 172.21.15.90:0/2614622726 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:19.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.676+0000 7f4625576700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fbab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 -- 172.21.15.90:0/2614622726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 msgr2=0x555826fbab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:19.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fbab60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 -- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 msgr2=0x555826fbc9a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:19.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fbc9a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 -- 172.21.15.90:0/2614622726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x555826d2ab60 con 0x555826fab780 2024-04-06T14:18:19.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f4624d75700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f4614003a70 tx=0x7f4614004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=588ae37d2a905c33 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:19.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f45fffff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:19.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4614022040 con 0x555826fab780 2024-04-06T14:18:19.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f4614027b00 con 0x555826fab780 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f461800f040 con 0x555826fab780 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4614027cc0 con 0x555826fab780 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f463318db80 10 client.17619 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.677+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555826d82cd0 con 0x555826fab780 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f461800f040 con 0x555826fab780 2024-04-06T14:18:19.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4614004c40 con 0x555826fab780 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 10 client.17619 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 10 client.17619 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 20 client.17619 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55582701aa70 0x55582701ce90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 4 client.17619 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x555826d4bcf0 con 0x55582701aa70 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f461ffff700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55582701aa70 0x55582701ce90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f45fffff700 10 client.17619 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f461ffff700 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55582701aa70 0x55582701ce90 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=31084b7beb06fb70 server_cookie=553a24956388b37 in_seq=0 out_seq=0 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.678+0000 7f45fffff700 1 -- 172.21.15.90:0/2614622726 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x555826d4bcf0 con 0x55582701aa70 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f45fffff700 10 client.17619 handle_command_reply: tid=0 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 client.17619 shutdown 2024-04-06T14:18:19.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 msgr2=0x555826fbc460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 secure :-1 s=READY pgs=816 cs=0 l=1 rev1=1 crypto rx=0x7f4614003a70 tx=0x7f4614004750 comp rx=0 tx=0).stop 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 shutdown_connections 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55582701aa70 0x55582701ce90 crc :-1 s=READY pgs=224 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555826fb5de0 0x555826fbab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555826fab780 0x555826fbc460 unknown :-1 s=CLOSED pgs=816 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 --2- 172.21.15.90:0/2614622726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555826fac0a0 0x555826fbc9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f45fffff700 0 client.17619 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 >> 172.21.15.90:0/2614622726 conn(0x555826f0a6a0 msgr2=0x555826fb1470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 shutdown_connections 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.679+0000 7f463318db80 1 -- 172.21.15.90:0/2614622726 wait complete. 2024-04-06T14:18:19.687 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.681+0000 7f463318db80 20 client.17619 trim_cache size 0 max 16384 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "7322034a-02d1-43d9-83d1-849396ab9680": { 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "7322034a-02d1-43d9-83d1-849396ab9680", 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:18:19.688 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 msgr2=0x7f46100a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 0x7f46100a3750 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4614005950 tx=0x7f461400b410 comp rx=0 tx=0).stop 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 msgr2=0x7f46201a39b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f460c009fd0 tx=0x7f460c00f9e0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 shutdown_connections 2024-04-06T14:18:19.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f46100a1290 0x7f46100a3750 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4620105c40 0x7f462019d670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f46200683a0 0x7f46201a3470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 --2- 172.21.15.90:0/4198966637 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4620103740 0x7f46201a39b0 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 >> 172.21.15.90:0/4198966637 conn(0x7f4620076d50 msgr2=0x7f46200fd180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 shutdown_connections 2024-04-06T14:18:19.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:19.683+0000 7f4626fd9700 1 -- 172.21.15.90:0/4198966637 wait complete. 2024-04-06T14:18:19.704 INFO:tasks.cephfs.filesystem:scrub status for tag:7322034a-02d1-43d9-83d1-849396ab9680 - {'path': '/', 'tag': '7322034a-02d1-43d9-83d1-849396ab9680', 'options': 'recursive,force'} 2024-04-06T14:18:19.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:19 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:18:19.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:19 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:19.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:19 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:18:19.993 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:19 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:20.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:19 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:18:20.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:19 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:20.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:19 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:18:20.247 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:19 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:20.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:19 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:18:20.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:19 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:20.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:19 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:18:20.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:19 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:21 smithi090 ceph-mon[43420]: pgmap v404: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 606 op/s 2024-04-06T14:18:21.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:21 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:18:21.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:21 smithi134 ceph-mon[47549]: pgmap v404: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 606 op/s 2024-04-06T14:18:21.398 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:21 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:18:21.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:21 smithi105 ceph-mon[46969]: pgmap v404: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 8.3 MiB/s wr, 606 op/s 2024-04-06T14:18:21.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:21 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:18:23.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 755.0 s, 1660.5 tps, lat 60.017 ms stddev 73.320 2024-04-06T14:18:23.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:23 smithi090 ceph-mon[43420]: pgmap v405: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 9.1 MiB/s wr, 678 op/s 2024-04-06T14:18:23.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:23 smithi090 ceph-mon[43420]: osdmap e98: 12 total, 12 up, 12 in 2024-04-06T14:18:23.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:23 smithi134 ceph-mon[47549]: pgmap v405: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 9.1 MiB/s wr, 678 op/s 2024-04-06T14:18:23.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:23 smithi134 ceph-mon[47549]: osdmap e98: 12 total, 12 up, 12 in 2024-04-06T14:18:23.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:23 smithi105 ceph-mon[46969]: pgmap v405: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 9.1 MiB/s wr, 678 op/s 2024-04-06T14:18:23.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:23 smithi105 ceph-mon[46969]: osdmap e98: 12 total, 12 up, 12 in 2024-04-06T14:18:25.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:25 smithi090 ceph-mon[43420]: pgmap v407: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 9.2 MiB/s wr, 647 op/s 2024-04-06T14:18:25.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:25 smithi134 ceph-mon[47549]: pgmap v407: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 9.2 MiB/s wr, 647 op/s 2024-04-06T14:18:25.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:25 smithi105 ceph-mon[46969]: pgmap v407: 97 pgs: 97 active+clean; 5.7 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 9.2 MiB/s wr, 647 op/s 2024-04-06T14:18:27.397 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:27 smithi134 ceph-mon[47549]: pgmap v408: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 8.9 MiB/s wr, 633 op/s 2024-04-06T14:18:27.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:27 smithi105 ceph-mon[46969]: pgmap v408: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 8.9 MiB/s wr, 633 op/s 2024-04-06T14:18:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:27 smithi090 ceph-mon[43420]: pgmap v408: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 8.9 MiB/s wr, 633 op/s 2024-04-06T14:18:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 760.0 s, 1330.0 tps, lat 75.013 ms stddev 89.866 2024-04-06T14:18:29.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:28 smithi090 ceph-mon[43420]: pgmap v409: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.4 MiB/s wr, 665 op/s 2024-04-06T14:18:29.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:29 smithi134 ceph-mon[47549]: pgmap v409: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.4 MiB/s wr, 665 op/s 2024-04-06T14:18:29.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:28 smithi105 ceph-mon[46969]: pgmap v409: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.4 MiB/s wr, 665 op/s 2024-04-06T14:18:31.399 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:31 smithi134 ceph-mon[47549]: pgmap v410: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.6 MiB/s wr, 686 op/s 2024-04-06T14:18:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:31 smithi105 ceph-mon[46969]: pgmap v410: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.6 MiB/s wr, 686 op/s 2024-04-06T14:18:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:31 smithi090 ceph-mon[43420]: pgmap v410: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 9.6 MiB/s wr, 686 op/s 2024-04-06T14:18:33.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 765.0 s, 1564.0 tps, lat 63.926 ms stddev 72.745 2024-04-06T14:18:33.402 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:33 smithi134 ceph-mon[47549]: pgmap v411: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.9 MiB/s wr, 699 op/s 2024-04-06T14:18:33.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:33 smithi105 ceph-mon[46969]: pgmap v411: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.9 MiB/s wr, 699 op/s 2024-04-06T14:18:33.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:33 smithi090 ceph-mon[43420]: pgmap v411: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.9 MiB/s wr, 699 op/s 2024-04-06T14:18:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:35 smithi134 ceph-mon[47549]: pgmap v412: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 9.2 MiB/s wr, 711 op/s 2024-04-06T14:18:35.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:35 smithi105 ceph-mon[46969]: pgmap v412: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 9.2 MiB/s wr, 711 op/s 2024-04-06T14:18:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:35 smithi090 ceph-mon[43420]: pgmap v412: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 9.2 MiB/s wr, 711 op/s 2024-04-06T14:18:37.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:37 smithi134 ceph-mon[47549]: pgmap v413: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.6 MiB/s wr, 689 op/s 2024-04-06T14:18:37.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:37 smithi105 ceph-mon[46969]: pgmap v413: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.6 MiB/s wr, 689 op/s 2024-04-06T14:18:37.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:37 smithi090 ceph-mon[43420]: pgmap v413: 97 pgs: 97 active+clean; 5.8 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.6 MiB/s wr, 689 op/s 2024-04-06T14:18:38.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 770.0 s, 1269.1 tps, lat 78.841 ms stddev 95.436 2024-04-06T14:18:39.251 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:38 smithi105 ceph-mon[46969]: pgmap v414: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 12 MiB/s wr, 811 op/s 2024-04-06T14:18:39.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:38 smithi090 ceph-mon[43420]: pgmap v414: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 12 MiB/s wr, 811 op/s 2024-04-06T14:18:39.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:38 smithi134 ceph-mon[47549]: pgmap v414: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 66 KiB/s rd, 12 MiB/s wr, 811 op/s 2024-04-06T14:18:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:41 smithi105 ceph-mon[46969]: pgmap v415: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 9.8 MiB/s wr, 668 op/s 2024-04-06T14:18:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:41 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:18:41.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:41 smithi090 ceph-mon[43420]: pgmap v415: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 9.8 MiB/s wr, 668 op/s 2024-04-06T14:18:41.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:41 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:18:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:41 smithi134 ceph-mon[47549]: pgmap v415: 97 pgs: 97 active+clean; 5.9 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 9.8 MiB/s wr, 668 op/s 2024-04-06T14:18:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:41 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:18:42.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:42 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:18:42.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:42 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:18:42.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:42 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:18:42.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:42 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:18:42.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:42 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:18:42.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:42 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:18:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:42 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:18:42.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:42 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:18:42.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:42 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:18:43.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 775.0 s, 821.4 tps, lat 120.304 ms stddev 196.445 2024-04-06T14:18:43.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:43 smithi105 ceph-mon[46969]: pgmap v416: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 21 MiB/s wr, 1.35k op/s 2024-04-06T14:18:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:43 smithi090 ceph-mon[43420]: pgmap v416: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 21 MiB/s wr, 1.35k op/s 2024-04-06T14:18:43.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:43 smithi134 ceph-mon[47549]: pgmap v416: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 21 MiB/s wr, 1.35k op/s 2024-04-06T14:18:45.505 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:45 smithi105 ceph-mon[46969]: pgmap v417: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 28 MiB/s wr, 1.76k op/s 2024-04-06T14:18:45.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:45 smithi090 ceph-mon[43420]: pgmap v417: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 28 MiB/s wr, 1.76k op/s 2024-04-06T14:18:45.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:45 smithi134 ceph-mon[47549]: pgmap v417: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 28 MiB/s wr, 1.76k op/s 2024-04-06T14:18:47.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:47 smithi105 ceph-mon[46969]: pgmap v418: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.68k op/s 2024-04-06T14:18:47.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:47 smithi090 ceph-mon[43420]: pgmap v418: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.68k op/s 2024-04-06T14:18:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:47 smithi134 ceph-mon[47549]: pgmap v418: 97 pgs: 97 active+clean; 6.1 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.68k op/s 2024-04-06T14:18:48.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 780.0 s, 1357.1 tps, lat 74.275 ms stddev 85.952 2024-04-06T14:18:49.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:49 smithi090 ceph-mon[43420]: pgmap v419: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 33 MiB/s wr, 2.02k op/s 2024-04-06T14:18:49.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:49 smithi134 ceph-mon[47549]: pgmap v419: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 33 MiB/s wr, 2.02k op/s 2024-04-06T14:18:49.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:49 smithi105 ceph-mon[46969]: pgmap v419: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 85 KiB/s rd, 33 MiB/s wr, 2.02k op/s 2024-04-06T14:18:49.706 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:18:49.706 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:18:50.033 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.033+0000 7fcff8647700 1 -- 172.21.15.90:0/3610308258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff010f4b0 msgr2=0x7fcff0107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.033+0000 7fcff8647700 1 --2- 172.21.15.90:0/3610308258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff010f4b0 0x7fcff0107db0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c0d0 tx=0x7fcfe801c3e0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.034+0000 7fcff8647700 1 -- 172.21.15.90:0/3610308258 shutdown_connections 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.034+0000 7fcff8647700 1 --2- 172.21.15.90:0/3610308258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff011a590 0x7fcff011c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.034+0000 7fcff8647700 1 --2- 172.21.15.90:0/3610308258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff010f4b0 0x7fcff0107db0 secure :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c0d0 tx=0x7fcfe801c3e0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.034+0000 7fcff8647700 1 --2- 172.21.15.90:0/3610308258 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff010f920 0x7fcff01082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.034+0000 7fcff8647700 1 -- 172.21.15.90:0/3610308258 >> 172.21.15.90:0/3610308258 conn(0x7fcff006be00 msgr2=0x7fcff006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.036+0000 7fcff8647700 1 -- 172.21.15.90:0/3610308258 shutdown_connections 2024-04-06T14:18:50.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.036+0000 7fcff8647700 1 -- 172.21.15.90:0/3610308258 wait complete. 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.036+0000 7fcff8647700 1 Processor -- start 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.036+0000 7fcff8647700 1 -- start start 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff010f920 0x7fcff0119d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff011a590 0x7fcff0115df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcff01cc230 con 0x7fcff010f920 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcff01cc370 con 0x7fcff0116330 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff8647700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcff01cc500 con 0x7fcff011a590 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff010f920 0x7fcff0119d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55562/0 (socket says 172.21.15.90:55562) 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 -- 172.21.15.90:0/2620449521 learned_addr learned my addr 172.21.15.90:0/2620449521 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff5be2700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff011a590 0x7fcff0115df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 -- 172.21.15.90:0/2620449521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff011a590 msgr2=0x7fcff0115df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff011a590 0x7fcff0115df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 -- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff010f920 msgr2=0x7fcff0119d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff010f920 0x7fcff0119d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 -- 172.21.15.90:0/2620449521 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfe801ad10 con 0x7fcff0116330 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.037+0000 7fcff6be4700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fcfec00b480 tx=0x7fcfec00b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.038+0000 7fcfe77fe700 1 -- 172.21.15.90:0/2620449521 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfec011830 con 0x7fcff0116330 2024-04-06T14:18:50.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.038+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcff01cc7e0 con 0x7fcff0116330 2024-04-06T14:18:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.038+0000 7fcfe77fe700 1 -- 172.21.15.90:0/2620449521 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfec00be90 con 0x7fcff0116330 2024-04-06T14:18:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.038+0000 7fcfe77fe700 1 -- 172.21.15.90:0/2620449521 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfec010d80 con 0x7fcff0116330 2024-04-06T14:18:50.039 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.038+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcff01ccca0 con 0x7fcff0116330 2024-04-06T14:18:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.040+0000 7fcfe77fe700 1 -- 172.21.15.90:0/2620449521 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fcfec01a400 con 0x7fcff0116330 2024-04-06T14:18:50.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.041+0000 7fcfe77fe700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 0x7fcfdc0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.041+0000 7fcfe77fe700 1 -- 172.21.15.90:0/2620449521 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcfec0cd4e0 con 0x7fcff0116330 2024-04-06T14:18:50.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.042+0000 7fcff63e3700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 0x7fcfdc0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.056 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.056+0000 7fcff63e3700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 0x7fcfdc0a3750 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fcfe8000f80 tx=0x7fcfe801c040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.057 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.058+0000 7fd0047fbb80 1 Processor -- start 2024-04-06T14:18:50.058 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.058+0000 7fd0047fbb80 1 -- start start 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b028a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b051d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40a50990 con 0x55ef40af90a0 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40a4e350 con 0x55ef40af8780 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40a49c40 con 0x55ef40b02de0 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff5be2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b028a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55568/0 (socket says 172.21.15.90:55568) 2024-04-06T14:18:50.059 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 -- 172.21.15.90:0/242343951 learned_addr learned my addr 172.21.15.90:0/242343951 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff6be4700 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b051d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 -- 172.21.15.90:0/242343951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 msgr2=0x55ef40b051d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b051d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 -- 172.21.15.90:0/242343951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 msgr2=0x55ef40b028a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b028a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 -- 172.21.15.90:0/242343951 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ef408cb3b0 con 0x55ef40af8780 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.059+0000 7fcff63e3700 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c0d0 tx=0x7fcfe800c000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c9dc9efa3f2341d0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fcfe4ff9700 1 -- 172.21.15.90:0/242343951 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfe800c070 con 0x55ef40af8780 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fcfe4ff9700 1 -- 172.21.15.90:0/242343951 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfe8018070 con 0x55ef40af8780 2024-04-06T14:18:50.060 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fcfe4ff9700 1 -- 172.21.15.90:0/242343951 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfe801f040 con 0x55ef40af8780 2024-04-06T14:18:50.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 -- 172.21.15.90:0/242343951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 msgr2=0x55ef40af8b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c0d0 tx=0x7fcfe800c000 comp rx=0 tx=0).stop 2024-04-06T14:18:50.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 -- 172.21.15.90:0/242343951 shutdown_connections 2024-04-06T14:18:50.064 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b051d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40af8b60 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/242343951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b028a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 -- 172.21.15.90:0/242343951 >> 172.21.15.90:0/242343951 conn(0x55ef40a576a0 msgr2=0x55ef40af3a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 -- 172.21.15.90:0/242343951 shutdown_connections 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.060+0000 7fd0047fbb80 1 -- 172.21.15.90:0/242343951 wait complete. 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 -- start start 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 Processor -- start 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40b02850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b001c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef409e5d80 con 0x55ef40af90a0 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef409e5a20 con 0x55ef40af8780 2024-04-06T14:18:50.065 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef408cd6a0 con 0x55ef40b02de0 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fd0047fbb80 10 client.0 _ll_register_callbacks cb 0x55ef409ef470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fcff5be2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40b02850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fcff5be2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36686/0 (socket says 172.21.15.90:36686) 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fcff5be2700 1 -- 172.21.15.90:0/905737785 learned_addr learned my addr 172.21.15.90:0/905737785 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.061+0000 7fcff6be4700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b001c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 -- 172.21.15.90:0/905737785 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 msgr2=0x55ef40b001c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b001c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 -- 172.21.15.90:0/905737785 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 msgr2=0x55ef40affad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 -- 172.21.15.90:0/905737785 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcff011d270 con 0x55ef40af8780 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff6be4700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b001c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:50.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff5be2700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcff63e3700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40b02850 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c420 tx=0x7fcfe80186b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=77a127498bb1bab3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcfd9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfe801a2e0 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ef40a0a380 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcfe80079c0 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcfe8018aa0 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fd0047fbb80 10 client.27350 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.062+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ef40877b60 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ef40a0a380 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcfe803c070 con 0x55ef40af8780 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 10 client.27350 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 10 client.27350 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 20 client.27350 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:50.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b125f0 0x55ef40b14a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 4 client.27350 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ef40a42700 con 0x55ef40b125f0 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fcff5be2700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b125f0 0x55ef40b14a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fcfd9ffb700 10 client.27350 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.063+0000 7fcff5be2700 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b125f0 0x55ef40b14a10 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9e6efb8e96d15265 server_cookie=d662f3aa9b4e1b2e in_seq=0 out_seq=0 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/905737785 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ef40a42700 con 0x55ef40b125f0 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fcfd9ffb700 10 client.27350 handle_command_reply: tid=0 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 client.27350 shutdown 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 msgr2=0x55ef40b02850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40b02850 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 crypto rx=0x7fcfe801c420 tx=0x7fcfe80186b0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 shutdown_connections 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b125f0 0x55ef40b14a10 crc :-1 s=READY pgs=225 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b001c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fcfd9ffb700 0 client.27350 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af8780 0x55ef40b02850 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/905737785 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40affad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 >> 172.21.15.90:0/905737785 conn(0x55ef40a576a0 msgr2=0x55ef40af2c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 shutdown_connections 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.064+0000 7fd0047fbb80 1 -- 172.21.15.90:0/905737785 wait complete. 2024-04-06T14:18:50.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.065+0000 7fd0047fbb80 20 client.27350 trim_cache size 0 max 16384 2024-04-06T14:18:50.090 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 Processor -- start 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 -- start start 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b221c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40af9a10 con 0x55ef40affc20 2024-04-06T14:18:50.091 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40af5190 con 0x55ef40af90a0 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40b22700 con 0x55ef40b02de0 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff6be4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff6be4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36698/0 (socket says 172.21.15.90:36698) 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55584/0 (socket says 172.21.15.90:55584) 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff6be4700 1 -- 172.21.15.90:0/222340607 learned_addr learned my addr 172.21.15.90:0/222340607 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.089+0000 7fcff5be2700 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b221c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 -- 172.21.15.90:0/222340607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 msgr2=0x55ef40b221c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b221c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 -- 172.21.15.90:0/222340607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 msgr2=0x55ef40b0eab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.092 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 -- 172.21.15.90:0/222340607 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ef40a0fea0 con 0x55ef40affc20 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff63e3700 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcff6be4700 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7fcfec0cadd0 tx=0x7fcfec0ba040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c363c07c065870c1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcfe57fa700 1 -- 172.21.15.90:0/222340607 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfec09a6e0 con 0x55ef40affc20 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcfe57fa700 1 -- 172.21.15.90:0/222340607 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfec09e510 con 0x55ef40affc20 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.090+0000 7fcfe57fa700 1 -- 172.21.15.90:0/222340607 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfec010580 con 0x55ef40affc20 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- 172.21.15.90:0/222340607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 msgr2=0x55ef40af33a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 secure :-1 s=READY pgs=817 cs=0 l=1 rev1=1 crypto rx=0x7fcfec0cadd0 tx=0x7fcfec0ba040 comp rx=0 tx=0).stop 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- 172.21.15.90:0/222340607 shutdown_connections 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40affc20 0x55ef40af33a0 unknown :-1 s=CLOSED pgs=817 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40af90a0 0x55ef40b0eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/222340607 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b02de0 0x55ef40b221c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.093 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- 172.21.15.90:0/222340607 >> 172.21.15.90:0/222340607 conn(0x55ef40a576a0 msgr2=0x55ef40aef890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- 172.21.15.90:0/222340607 shutdown_connections 2024-04-06T14:18:50.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- 172.21.15.90:0/222340607 wait complete. 2024-04-06T14:18:50.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- start start 2024-04-06T14:18:50.094 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 Processor -- start 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b220a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 0x55ef40b205d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b20b10 0x55ef40b20f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40b68bd0 con 0x55ef40af90a0 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40b68d40 con 0x55ef40b02de0 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ef40b68f00 con 0x55ef40b20b10 2024-04-06T14:18:50.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.091+0000 7fd0047fbb80 10 client.0 _ll_register_callbacks cb 0x55ef409ef470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff5be2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 0x55ef40b205d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b220a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff6be4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b20b10 0x55ef40b20f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff5be2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 0x55ef40b205d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55598/0 (socket says 172.21.15.90:55598) 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff5be2700 1 -- 172.21.15.90:0/453650873 learned_addr learned my addr 172.21.15.90:0/453650873 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 -- 172.21.15.90:0/453650873 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b20b10 msgr2=0x55ef40b20f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b20b10 0x55ef40b20f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 -- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 msgr2=0x55ef40b205d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 0x55ef40b205d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 -- 172.21.15.90:0/453650873 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ef40877b60 con 0x55ef40af90a0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcff63e3700 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b220a0 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7fcfe8035d30 tx=0x7fcfe8014950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8bcbcb63b1ff1033 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcfd9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfe801f040 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcfe8014470 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fcfe8018070 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcfe8018da0 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fd0047fbb80 10 client.17634 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.092+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fcff01ccca0 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfe8018070 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcfe8004cf0 con 0x55ef40af90a0 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 10 client.17634 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 10 client.17634 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 20 client.17634 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:50.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b6a280 0x55ef40b6c670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 4 client.17634 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ef40898cf0 con 0x55ef40b6a280 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.093+0000 7fcff5be2700 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b6a280 0x55ef40b6c670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fcff5be2700 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b6a280 0x55ef40b6c670 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5eacf0fddd510b3 server_cookie=1a0cd38017f04105 in_seq=0 out_seq=0 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fcfd9ffb700 10 client.17634 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fcfd9ffb700 1 -- 172.21.15.90:0/453650873 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55ef40898cf0 con 0x55ef40b6a280 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fcfd9ffb700 10 client.17634 handle_command_reply: tid=0 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fd0047fbb80 1 client.17634 shutdown 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 msgr2=0x55ef40b220a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.094+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b220a0 secure :-1 s=READY pgs=818 cs=0 l=1 rev1=1 crypto rx=0x7fcfe8035d30 tx=0x7fcfe8014950 comp rx=0 tx=0).stop 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 shutdown_connections 2024-04-06T14:18:50.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ef40b6a280 0x55ef40b6c670 crc :-1 s=READY pgs=226 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ef40b20b10 0x55ef40b20f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ef40af90a0 0x55ef40b220a0 unknown :-1 s=CLOSED pgs=818 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fcfd9ffb700 0 client.17634 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 --2- 172.21.15.90:0/453650873 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ef40b02de0 0x55ef40b205d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 >> 172.21.15.90:0/453650873 conn(0x55ef40a576a0 msgr2=0x55ef40afe3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 shutdown_connections 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.095+0000 7fd0047fbb80 1 -- 172.21.15.90:0/453650873 wait complete. 2024-04-06T14:18:50.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fd0047fbb80 20 client.17634 trim_cache size 0 max 16384 2024-04-06T14:18:50.101 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:50.101 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:18:50.101 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:18:50.101 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 msgr2=0x7fcfdc0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 0x7fcfdc0a3750 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fcfe8000f80 tx=0x7fcfe801c040 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 msgr2=0x7fcff01167e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7fcfec00b480 tx=0x7fcfec00b790 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 shutdown_connections 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcfdc0a1290 0x7fcfdc0a3750 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcff0116330 0x7fcff01167e0 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcff010f920 0x7fcff0119d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 --2- 172.21.15.90:0/2620449521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcff011a590 0x7fcff0115df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 >> 172.21.15.90:0/2620449521 conn(0x7fcff006be00 msgr2=0x7fcff010c0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 shutdown_connections 2024-04-06T14:18:50.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.096+0000 7fcff8647700 1 -- 172.21.15.90:0/2620449521 wait complete. 2024-04-06T14:18:50.112 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:18:50.112 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:18:50.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 -- 172.21.15.90:0/395002913 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073a70 msgr2=0x7f78c81116d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 --2- 172.21.15.90:0/395002913 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073a70 0x7f78c81116d0 secure :-1 s=READY pgs=819 cs=0 l=1 rev1=1 crypto rx=0x7f78b8009990 tx=0x7f78b8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 -- 172.21.15.90:0/395002913 shutdown_connections 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 --2- 172.21.15.90:0/395002913 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c8114000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 --2- 172.21.15.90:0/395002913 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073150 0x7f78c8073530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 --2- 172.21.15.90:0/395002913 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073a70 0x7f78c81116d0 unknown :-1 s=CLOSED pgs=819 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 -- 172.21.15.90:0/395002913 >> 172.21.15.90:0/395002913 conn(0x7f78c80fc730 msgr2=0x7f78c80feb50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 -- 172.21.15.90:0/395002913 shutdown_connections 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.318+0000 7f78cea52700 1 -- 172.21.15.90:0/395002913 wait complete. 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.319+0000 7f78cea52700 1 Processor -- start 2024-04-06T14:18:50.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.319+0000 7f78cea52700 1 -- start start 2024-04-06T14:18:50.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073a70 0x7f78c819dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c81a0fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78c81a15e0 con 0x7f78c8073150 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78c81a1750 con 0x7f78c8073a70 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78cea52700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78c81a1910 con 0x7f78c8111c10 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36718/0 (socket says 172.21.15.90:36718) 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 -- 172.21.15.90:0/2316116340 learned_addr learned my addr 172.21.15.90:0/2316116340 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c77fe700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073a70 0x7f78c819dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78ccfef700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c81a0fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 -- 172.21.15.90:0/2316116340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 msgr2=0x7f78c81a0fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c81a0fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 -- 172.21.15.90:0/2316116340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073a70 msgr2=0x7f78c819dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073a70 0x7f78c819dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 -- 172.21.15.90:0/2316116340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78c0009c40 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78ccfef700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c81a0fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c7fff700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f78c000c370 tx=0x7f78c000c680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.320+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78c0017070 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.321+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78b80095f0 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.321+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f78c000f040 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.321+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78c000e050 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.321+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78c81aa370 con 0x7f78c8073150 2024-04-06T14:18:50.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.322+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78c8104130 con 0x7f78c8073150 2024-04-06T14:18:50.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.324+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f78c00044e0 con 0x7f78c8073150 2024-04-06T14:18:50.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.324+0000 7f78c57fa700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 0x7f78b40a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.325+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f78c00cc300 con 0x7f78c8073150 2024-04-06T14:18:50.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.328+0000 7f78c77fe700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 0x7f78b40a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.329+0000 7f78c77fe700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 0x7f78b40a3750 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f78c819ed00 tx=0x7f78b8005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.329+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f78c0095730 con 0x7f78c8073150 2024-04-06T14:18:50.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.735+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f78c8074890 con 0x7f78c8073150 2024-04-06T14:18:50.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.736+0000 7f78c57fa700 1 -- 172.21.15.90:0/2316116340 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f78c0098d50 con 0x7f78c8073150 2024-04-06T14:18:50.736 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:18:50.736 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:18:50.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 msgr2=0x7f78b40a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 0x7f78b40a3750 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f78c819ed00 tx=0x7f78b8005fd0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 msgr2=0x7f78c819d7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 secure :-1 s=READY pgs=820 cs=0 l=1 rev1=1 crypto rx=0x7f78c000c370 tx=0x7f78c000c680 comp rx=0 tx=0).stop 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 shutdown_connections 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f78b40a1290 0x7f78b40a3750 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f78c8111c10 0x7f78c81a0fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f78c8073150 0x7f78c819d7b0 unknown :-1 s=CLOSED pgs=820 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 --2- 172.21.15.90:0/2316116340 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f78c8073a70 0x7f78c819dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 >> 172.21.15.90:0/2316116340 conn(0x7f78c80fc730 msgr2=0x7f78c80fe040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 shutdown_connections 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.739+0000 7f78cea52700 1 -- 172.21.15.90:0/2316116340 wait complete. 2024-04-06T14:18:50.739 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:18:50.758 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:50.758 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1811188104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 msgr2=0x7f7b9810dc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1811188104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 0x7f7b9810dc10 secure :-1 s=READY pgs=821 cs=0 l=1 rev1=1 crypto rx=0x7f7b9000b0a0 tx=0x7f7b9000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1811188104 shutdown_connections 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1811188104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 0x7f7b9810dc10 unknown :-1 s=CLOSED pgs=821 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1811188104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b98100f70 0x7f7b98101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1811188104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98101920 0x7f7b98105960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.945 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1811188104 >> 172.21.15.90:0/1811188104 conn(0x7f7b980fc7f0 msgr2=0x7f7b980fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1811188104 shutdown_connections 2024-04-06T14:18:50.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.945+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1811188104 wait complete. 2024-04-06T14:18:50.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 Processor -- start 2024-04-06T14:18:50.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 -- start start 2024-04-06T14:18:50.946 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 0x7f7b98197340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 0x7f7b98197880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 0x7f7b9819c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b9819c6a0 con 0x7f7b98106090 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b9819c810 con 0x7f7b9819bc20 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.946+0000 7f7b9edb3700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b9819c9d0 con 0x7f7b98100f70 2024-04-06T14:18:50.947 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.948+0000 7f7b9d350700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 0x7f7b9819c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.948+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 0x7f7b98197340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.948+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 0x7f7b98197340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55734/0 (socket says 172.21.15.90:55734) 2024-04-06T14:18:50.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.948+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/1151760266 learned_addr learned my addr 172.21.15.90:0/1151760266 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.948 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 -- 172.21.15.90:0/1151760266 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 msgr2=0x7f7b98197340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 0x7f7b98197340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 -- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 msgr2=0x7f7b98197880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 0x7f7b98197880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 -- 172.21.15.90:0/1151760266 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b8c009520 con 0x7f7b9819bc20 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b9d350700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 0x7f7b9819c0a0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f7b9000b070 tx=0x7f7b9000bcf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.949 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.949+0000 7f7b95ffb700 1 -- 172.21.15.90:0/1151760266 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b9000e040 con 0x7f7b9819bc20 2024-04-06T14:18:50.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.950+0000 7f7b95ffb700 1 -- 172.21.15.90:0/1151760266 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7b90007e10 con 0x7f7b9819bc20 2024-04-06T14:18:50.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.950+0000 7f7b95ffb700 1 -- 172.21.15.90:0/1151760266 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b900052a0 con 0x7f7b9819bc20 2024-04-06T14:18:50.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.950+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7b90009d00 con 0x7f7b9819bc20 2024-04-06T14:18:50.950 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.950+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b98073220 con 0x7f7b9819bc20 2024-04-06T14:18:50.951 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.952+0000 7f7b95ffb700 1 -- 172.21.15.90:0/1151760266 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f7b9001d480 con 0x7f7b9819bc20 2024-04-06T14:18:50.952 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.953+0000 7f7b95ffb700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 0x7f7b880a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.953+0000 7f7b95ffb700 1 -- 172.21.15.90:0/1151760266 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7b900cd920 con 0x7f7b9819bc20 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.953+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 0x7f7b880a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.953+0000 7f7baaf67b80 1 Processor -- start 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 -- start start 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 0x7f7b880a37d0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c009ef0 tx=0x7f7b8c009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b068a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.954 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b06de0 0x55eeb7b091d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7a54990 con 0x55eeb7afd0a0 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7a52350 con 0x55eeb7b06de0 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7a4dc40 con 0x55eeb7afc780 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55744/0 (socket says 172.21.15.90:55744) 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/1132455339 learned_addr learned my addr 172.21.15.90:0/1132455339 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b97fff700 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b068a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9d350700 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b06de0 0x55eeb7b091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/1132455339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b06de0 msgr2=0x55eeb7b091d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b06de0 0x55eeb7b091d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/1132455339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 msgr2=0x55eeb7b068a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b068a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.955 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/1132455339 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eeb78cf3b0 con 0x55eeb7afc780 2024-04-06T14:18:50.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.954+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c00f6d0 tx=0x7f7b8c014000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9876f2507063a4b4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.955+0000 7f7b82ffd700 1 -- 172.21.15.90:0/1132455339 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c004220 con 0x55eeb7afc780 2024-04-06T14:18:50.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.955+0000 7f7b82ffd700 1 -- 172.21.15.90:0/1132455339 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7b8c021070 con 0x55eeb7afc780 2024-04-06T14:18:50.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.955+0000 7f7b82ffd700 1 -- 172.21.15.90:0/1132455339 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c0266e0 con 0x55eeb7afc780 2024-04-06T14:18:50.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.955+0000 7f7baaf67b80 1 -- 172.21.15.90:0/1132455339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 msgr2=0x55eeb7afcb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.955+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c00f6d0 tx=0x7f7b8c014000 comp rx=0 tx=0).stop 2024-04-06T14:18:50.959 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 -- 172.21.15.90:0/1132455339 shutdown_connections 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b06de0 0x55eeb7b091d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afc780 0x55eeb7afcb60 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/1132455339 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b068a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 -- 172.21.15.90:0/1132455339 >> 172.21.15.90:0/1132455339 conn(0x55eeb7a5b6a0 msgr2=0x55eeb7af7a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 -- 172.21.15.90:0/1132455339 shutdown_connections 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 -- 172.21.15.90:0/1132455339 wait complete. 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 -- start start 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.956+0000 7f7baaf67b80 1 Processor -- start 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b03b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b04250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb79e9d80 con 0x55eeb7afd0a0 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb79e9a20 con 0x55eeb7afc780 2024-04-06T14:18:50.960 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb78d16a0 con 0x55eeb7b06de0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 10 client.0 _ll_register_callbacks cb 0x55eeb79f3470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55640/0 (socket says 172.21.15.90:55640) 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/2211402664 learned_addr learned my addr 172.21.15.90:0/2211402664 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9d350700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b04250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b97fff700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b03b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/2211402664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 msgr2=0x55eeb7b04250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b04250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/2211402664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 msgr2=0x55eeb7b03b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b03b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/2211402664 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b8c017070 con 0x55eeb7afc780 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c00f6a0 tx=0x7f7b8c00fe70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=592e1edae1205e61 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b77fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:50.961 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c003fd0 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55eeb7a0e380 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.957+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7b8c004130 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7b8c0042f0 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 10 client.27374 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55eeb787bb60 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7b8c03c080 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7b8c030440 con 0x55eeb7afc780 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 10 client.27374 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 10 client.27374 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 20 client.27374 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b16880 0x55eeb7b18ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 4 client.27374 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55eeb7a46700 con 0x55eeb7b16880 2024-04-06T14:18:50.962 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.958+0000 7f7b97fff700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b16880 0x55eeb7b18ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7b77fff700 10 client.27374 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7b97fff700 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b16880 0x55eeb7b18ca0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=eef3ee3ee570e274 server_cookie=3ef27367c32457ee in_seq=0 out_seq=0 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7b77fff700 1 -- 172.21.15.90:0/2211402664 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55eeb7a46700 con 0x55eeb7b16880 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7b77fff700 10 client.27374 handle_command_reply: tid=0 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 client.27374 shutdown 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 msgr2=0x55eeb7b06840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c00f6a0 tx=0x7f7b8c00fe70 comp rx=0 tx=0).stop 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 shutdown_connections 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b16880 0x55eeb7b18ca0 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.959+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b04250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b06840 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/2211402664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b03b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7b77fff700 0 client.27374 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:50.963 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 >> 172.21.15.90:0/2211402664 conn(0x55eeb7a5b6a0 msgr2=0x55eeb7af6c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 shutdown_connections 2024-04-06T14:18:50.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 1 -- 172.21.15.90:0/2211402664 wait complete. 2024-04-06T14:18:50.964 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.960+0000 7f7baaf67b80 20 client.27374 trim_cache size 0 max 16384 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 Processor -- start 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 -- start start 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b03ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b0da60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b05100 con 0x55eeb7b06de0 2024-04-06T14:18:50.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b0dfa0 con 0x55eeb7afc780 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b19300 con 0x55eeb7afd0a0 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b9d350700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b9d350700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36762/0 (socket says 172.21.15.90:36762) 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b0da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b9d350700 1 -- 172.21.15.90:0/4130607840 learned_addr learned my addr 172.21.15.90:0/4130607840 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b03ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 -- 172.21.15.90:0/4130607840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 msgr2=0x55eeb7b03ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b03ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 -- 172.21.15.90:0/4130607840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 msgr2=0x55eeb7b06370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b97fff700 1 -- 172.21.15.90:0/4130607840 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55eeb7a13ea0 con 0x55eeb7afd0a0 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.981+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b03ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:50.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.980+0000 7f7b9d350700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.981+0000 7f7b97fff700 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b0da60 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x55eeb7b19510 tx=0x7f7b84004320 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2f65a27e35882656 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.981+0000 7f7b837fe700 1 -- 172.21.15.90:0/4130607840 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84016040 con 0x55eeb7afd0a0 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.981+0000 7f7b837fe700 1 -- 172.21.15.90:0/4130607840 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7b8401b5a0 con 0x55eeb7afd0a0 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.981+0000 7f7b837fe700 1 -- 172.21.15.90:0/4130607840 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b84005020 con 0x55eeb7afd0a0 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- 172.21.15.90:0/4130607840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 msgr2=0x55eeb7b0da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b0da60 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x55eeb7b19510 tx=0x7f7b84004320 comp rx=0 tx=0).stop 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- 172.21.15.90:0/4130607840 shutdown_connections 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7b06de0 0x55eeb7b06370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7afc780 0x55eeb7b03ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/4130607840 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7afd0a0 0x55eeb7b0da60 secure :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0x55eeb7b19510 tx=0x7f7b84004320 comp rx=0 tx=0).stop 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- 172.21.15.90:0/4130607840 >> 172.21.15.90:0/4130607840 conn(0x55eeb7a5b6a0 msgr2=0x55eeb7b033a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- 172.21.15.90:0/4130607840 shutdown_connections 2024-04-06T14:18:50.983 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- 172.21.15.90:0/4130607840 wait complete. 2024-04-06T14:18:50.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 -- start start 2024-04-06T14:18:50.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.982+0000 7f7baaf67b80 1 Processor -- start 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b0bc60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 0x55eeb7b0c620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b6a710 con 0x55eeb7afc780 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b6a880 con 0x55eeb7b0c1a0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55eeb7b7ba50 con 0x55eeb7b06de0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7baaf67b80 10 client.0 _ll_register_callbacks cb 0x55eeb79f3470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7b9cb4f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36768/0 (socket says 172.21.15.90:36768) 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/3254066952 learned_addr learned my addr 172.21.15.90:0/3254066952 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.983+0000 7f7b9d350700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 0x55eeb7b0c620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b97fff700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b0bc60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/3254066952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 msgr2=0x55eeb7b0bc60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b0bc60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 msgr2=0x55eeb7b0c620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 0x55eeb7b0c620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 -- 172.21.15.90:0/3254066952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55eeb787bb60 con 0x55eeb7afc780 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b97fff700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b0bc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9d350700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 0x55eeb7b0c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b9cb4f700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c005810 tx=0x7f7b8c00fe00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d51812096a337e51 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b77fff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7b8c005450 con 0x55eeb7afc780 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f7b8c0055b0 con 0x55eeb7afc780 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7b8c021070 con 0x55eeb7afc780 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7b8c030b70 con 0x55eeb7afc780 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7baaf67b80 10 client.17655 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:50.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f7b98073220 con 0x55eeb7afc780 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.984+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7b8c021070 con 0x55eeb7afc780 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f7b8c042020 con 0x55eeb7afc780 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 10 client.17655 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 10 client.17655 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 20 client.17655 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b7cf20 0x55eeb7b7f310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 4 client.17655 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55eeb789ccf0 con 0x55eeb7b7cf20 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b97fff700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b7cf20 0x55eeb7b7f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b77fff700 10 client.17655 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b97fff700 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b7cf20 0x55eeb7b7f310 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e2ed4ece87f02e19 server_cookie=10ad8a311fe157ec in_seq=0 out_seq=0 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b77fff700 1 -- 172.21.15.90:0/3254066952 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55eeb789ccf0 con 0x55eeb7b7cf20 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7b77fff700 10 client.17655 handle_command_reply: tid=0 2024-04-06T14:18:50.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.985+0000 7f7baaf67b80 1 client.17655 shutdown 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 msgr2=0x55eeb7b0d8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 secure :-1 s=READY pgs=822 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c005810 tx=0x7f7b8c00fe00 comp rx=0 tx=0).stop 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 shutdown_connections 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55eeb7b7cf20 0x55eeb7b7f310 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55eeb7b0c1a0 0x55eeb7b0c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55eeb7afc780 0x55eeb7b0d8c0 unknown :-1 s=CLOSED pgs=822 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 --2- 172.21.15.90:0/3254066952 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55eeb7b06de0 0x55eeb7b0bc60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 >> 172.21.15.90:0/3254066952 conn(0x55eeb7a5b6a0 msgr2=0x55eeb7afc000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.986+0000 7f7b77fff700 0 client.17655 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:18:50.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.988+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 shutdown_connections 2024-04-06T14:18:50.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.988+0000 7f7baaf67b80 1 -- 172.21.15.90:0/3254066952 wait complete. 2024-04-06T14:18:50.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.988+0000 7f7baaf67b80 20 client.17655 trim_cache size 0 max 16384 2024-04-06T14:18:50.991 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 msgr2=0x7f7b880a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 0x7f7b880a37d0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f7b8c009ef0 tx=0x7f7b8c009260 comp rx=0 tx=0).stop 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 msgr2=0x7f7b9819c0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 0x7f7b9819c0a0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 crypto rx=0x7f7b9000b070 tx=0x7f7b9000bcf0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 shutdown_connections 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f7b880a1310 0x7f7b880a37d0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b9819bc20 0x7f7b9819c0a0 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f7b98100f70 0x7f7b98197340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 --2- 172.21.15.90:0/1151760266 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f7b98106090 0x7f7b98197880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:50.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 >> 172.21.15.90:0/1151760266 conn(0x7f7b980fc7f0 msgr2=0x7f7b980feb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:50.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 shutdown_connections 2024-04-06T14:18:50.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:50.989+0000 7f7b9edb3700 1 -- 172.21.15.90:0/1151760266 wait complete. 2024-04-06T14:18:51.035 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:51.035 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:18:51.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.275+0000 7ff54b925700 1 -- 172.21.15.90:0/1884044541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff544109ee0 msgr2=0x7ff5441163a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.275+0000 7ff54b925700 1 --2- 172.21.15.90:0/1884044541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff544109ee0 0x7ff5441163a0 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 crypto rx=0x7ff53c009920 tx=0x7ff53c009c30 comp rx=0 tx=0).stop 2024-04-06T14:18:51.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.278+0000 7ff54b925700 1 -- 172.21.15.90:0/1884044541 shutdown_connections 2024-04-06T14:18:51.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.278+0000 7ff54b925700 1 --2- 172.21.15.90:0/1884044541 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff544109ee0 0x7ff5441163a0 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.278 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.278+0000 7ff54b925700 1 --2- 172.21.15.90:0/1884044541 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff54410f5e0 0x7ff54410f9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.278+0000 7ff54b925700 1 --2- 172.21.15.90:0/1884044541 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff544109520 0x7ff5441099a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.279 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.278+0000 7ff54b925700 1 -- 172.21.15.90:0/1884044541 >> 172.21.15.90:0/1884044541 conn(0x7ff544072290 msgr2=0x7ff5440726a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.281+0000 7ff54b925700 1 -- 172.21.15.90:0/1884044541 shutdown_connections 2024-04-06T14:18:51.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.281+0000 7ff54b925700 1 -- 172.21.15.90:0/1884044541 wait complete. 2024-04-06T14:18:51.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.281+0000 7ff54b925700 1 Processor -- start 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.281+0000 7ff54b925700 1 -- start start 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff544109520 0x7ff5441ae770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff54410f5e0 0x7ff5441b1fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5441b25a0 con 0x7ff544109520 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5441b2710 con 0x7ff54410f5e0 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff54b925700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff5441b28d0 con 0x7ff544109ee0 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55776/0 (socket says 172.21.15.90:55776) 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 -- 172.21.15.90:0/1161096579 learned_addr learned my addr 172.21.15.90:0/1161096579 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff549ec2700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff54410f5e0 0x7ff5441b1fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 -- 172.21.15.90:0/1161096579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff54410f5e0 msgr2=0x7ff5441b1fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff54410f5e0 0x7ff5441b1fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 -- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff544109520 msgr2=0x7ff5441ae770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff544109520 0x7ff5441ae770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.282+0000 7ff548ec0700 1 -- 172.21.15.90:0/1161096579 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff53c009520 con 0x7ff544109ee0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff548ec0700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7ff534009a80 tx=0x7ff534009d90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff53a7fc700 1 -- 172.21.15.90:0/1161096579 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff53400cb70 con 0x7ff544109ee0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff5441b2bb0 con 0x7ff544109ee0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff5441bb510 con 0x7ff544109ee0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff53a7fc700 1 -- 172.21.15.90:0/1161096579 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff53400f4d0 con 0x7ff544109ee0 2024-04-06T14:18:51.285 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.283+0000 7ff53a7fc700 1 -- 172.21.15.90:0/1161096579 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff534020610 con 0x7ff544109ee0 2024-04-06T14:18:51.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.285+0000 7ff53a7fc700 1 -- 172.21.15.90:0/1161096579 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff53400ccd0 con 0x7ff544109ee0 2024-04-06T14:18:51.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.286+0000 7ff53a7fc700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 0x7ff5300a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.286 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.286+0000 7ff53a7fc700 1 -- 172.21.15.90:0/1161096579 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff5340cc9a0 con 0x7ff544109ee0 2024-04-06T14:18:51.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.287+0000 7ff5496c1700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 0x7ff5300a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 Processor -- start 2024-04-06T14:18:51.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff5496c1700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 0x7ff5300a3790 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7ff5400062a0 tx=0x7ff5400061f0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 -- start start 2024-04-06T14:18:51.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.289 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2bf0a0 0x55733e2c88a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cb1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e216990 con 0x55733e2c8de0 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e214350 con 0x55733e2bf0a0 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e20fc40 con 0x55733e2be780 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55788/0 (socket says 172.21.15.90:55788) 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff5496c1700 1 -- 172.21.15.90:0/2200364959 learned_addr learned my addr 172.21.15.90:0/2200364959 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.289+0000 7ff549ec2700 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cb1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff548ec0700 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2bf0a0 0x55733e2c88a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 -- 172.21.15.90:0/2200364959 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2bf0a0 msgr2=0x55733e2c88a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2bf0a0 0x55733e2c88a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 -- 172.21.15.90:0/2200364959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 msgr2=0x55733e2cb1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.290 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cb1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 -- 172.21.15.90:0/2200364959 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55733e0913b0 con 0x55733e2be780 2024-04-06T14:18:51.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7ff540009610 tx=0x7ff540010000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6816d18ec3d631b4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff52f7fe700 1 -- 172.21.15.90:0/2200364959 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54001c070 con 0x55733e2be780 2024-04-06T14:18:51.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff52f7fe700 1 -- 172.21.15.90:0/2200364959 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff54000f040 con 0x55733e2be780 2024-04-06T14:18:51.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.290+0000 7ff52f7fe700 1 -- 172.21.15.90:0/2200364959 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff540020670 con 0x55733e2be780 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2200364959 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 msgr2=0x55733e2beb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7ff540009610 tx=0x7ff540010000 comp rx=0 tx=0).stop 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2200364959 shutdown_connections 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cb1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2be780 0x55733e2beb60 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2200364959 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2bf0a0 0x55733e2c88a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2200364959 >> 172.21.15.90:0/2200364959 conn(0x55733e21d6a0 msgr2=0x55733e2b9a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2200364959 shutdown_connections 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2200364959 wait complete. 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.291+0000 7ff557ad9b80 1 -- start start 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 Processor -- start 2024-04-06T14:18:51.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2c5b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2c8de0 0x55733e2c6160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e1abd80 con 0x55733e2be780 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e1aba20 con 0x55733e2c8de0 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e0936a0 con 0x55733e2bf0a0 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff557ad9b80 10 client.0 _ll_register_callbacks cb 0x55733e1b5470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36796/0 (socket says 172.21.15.90:36796) 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff5496c1700 1 -- 172.21.15.90:0/983067004 learned_addr learned my addr 172.21.15.90:0/983067004 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.292+0000 7ff549ec2700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2c8de0 0x55733e2c6160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 -- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 msgr2=0x55733e2c5b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2c5b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 -- 172.21.15.90:0/983067004 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2c8de0 msgr2=0x55733e2c6160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2c8de0 0x55733e2c6160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 -- 172.21.15.90:0/983067004 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff540007c30 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff5496c1700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7ff540009080 tx=0x7ff540017ef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7351e42abd6fb3f7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff52cff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54001c040 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff540015820 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55733e1d0380 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.293+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff540017d40 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 10 client.17664 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55733e03db60 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55733e1d0380 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff540020be0 con 0x55733e2be780 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 10 client.17664 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:51.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 10 client.17664 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 20 client.17664 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e2d88e0 0x55733e2dad00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 4 client.17664 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55733e208700 con 0x55733e2d88e0 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.294+0000 7ff548ec0700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e2d88e0 0x55733e2dad00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.295+0000 7ff548ec0700 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e2d88e0 0x55733e2dad00 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=cfa16d9d9fc4ad7e server_cookie=d3c5eb0ffe828c57 in_seq=0 out_seq=0 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.295+0000 7ff52cff9700 10 client.17664 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.295+0000 7ff52cff9700 1 -- 172.21.15.90:0/983067004 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55733e208700 con 0x55733e2d88e0 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.295+0000 7ff52cff9700 10 client.17664 handle_command_reply: tid=0 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.295+0000 7ff557ad9b80 1 client.17664 shutdown 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 msgr2=0x55733e2c87f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 secure :-1 s=READY pgs=823 cs=0 l=1 rev1=1 crypto rx=0x7ff540009080 tx=0x7ff540017ef0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 shutdown_connections 2024-04-06T14:18:51.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e2d88e0 0x55733e2dad00 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2c8de0 0x55733e2c6160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2be780 0x55733e2c87f0 unknown :-1 s=CLOSED pgs=823 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/983067004 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2c5b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 >> 172.21.15.90:0/983067004 conn(0x55733e21d6a0 msgr2=0x55733e2b8c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff52cff9700 0 client.17664 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 shutdown_connections 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 1 -- 172.21.15.90:0/983067004 wait complete. 2024-04-06T14:18:51.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.296+0000 7ff557ad9b80 20 client.17664 trim_cache size 0 max 16384 2024-04-06T14:18:51.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.312+0000 7ff557ad9b80 1 Processor -- start 2024-04-06T14:18:51.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 -- start start 2024-04-06T14:18:51.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2badc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2bf0a0 0x55733e2cfa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e2b1330 con 0x55733e2bf0a0 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e2cffd0 con 0x55733e2be780 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e2bffe0 con 0x55733e2c8de0 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff549ec2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.313+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2badc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55796/0 (socket says 172.21.15.90:55796) 2024-04-06T14:18:51.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 -- 172.21.15.90:0/3684082262 learned_addr learned my addr 172.21.15.90:0/3684082262 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff548ec0700 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2bf0a0 0x55733e2cfa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 -- 172.21.15.90:0/3684082262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 msgr2=0x55733e2badc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2badc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 -- 172.21.15.90:0/3684082262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2bf0a0 msgr2=0x55733e2cfa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2bf0a0 0x55733e2cfa90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 -- 172.21.15.90:0/3684082262 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55733e1d5ea0 con 0x55733e2c8de0 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff549ec2700 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7ff53c01f040 tx=0x7ff53c016040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=134d5e9f2bc90c69 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff52ffff700 1 -- 172.21.15.90:0/3684082262 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff53c01c700 con 0x55733e2c8de0 2024-04-06T14:18:51.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff52ffff700 1 -- 172.21.15.90:0/3684082262 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff53c00b650 con 0x55733e2c8de0 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.314+0000 7ff52ffff700 1 -- 172.21.15.90:0/3684082262 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff53c005510 con 0x55733e2c8de0 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- 172.21.15.90:0/3684082262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 msgr2=0x55733e2c8320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7ff53c01f040 tx=0x7ff53c016040 comp rx=0 tx=0).stop 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- 172.21.15.90:0/3684082262 shutdown_connections 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2c8de0 0x55733e2c8320 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2badc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/3684082262 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2bf0a0 0x55733e2cfa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- 172.21.15.90:0/3684082262 >> 172.21.15.90:0/3684082262 conn(0x55733e21d6a0 msgr2=0x55733e2c4400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- 172.21.15.90:0/3684082262 shutdown_connections 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- 172.21.15.90:0/3684082262 wait complete. 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 -- start start 2024-04-06T14:18:51.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.315+0000 7ff557ad9b80 1 Processor -- start 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2cd800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cdd40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e32b890 con 0x55733e2c8de0 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e32ba00 con 0x55733e2be780 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55733e32bbc0 con 0x55733e2bf0a0 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff557ad9b80 10 client.0 _ll_register_callbacks cb 0x55733e1b5470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55714/0 (socket says 172.21.15.90:55714) 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 -- 172.21.15.90:0/2705525044 learned_addr learned my addr 172.21.15.90:0/2705525044 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff549ec2700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cdd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 -- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 msgr2=0x55733e2cd800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2cd800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 -- 172.21.15.90:0/2705525044 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 msgr2=0x55733e2cdd40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cdd40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.316+0000 7ff5496c1700 1 -- 172.21.15.90:0/2705525044 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff544116ca0 con 0x55733e2be780 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff5496c1700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7ff540009610 tx=0x7ff540015ee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=809a1964078bd25d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff52cff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff54001c070 con 0x55733e2be780 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7ff5400317a0 con 0x55733e2be780 2024-04-06T14:18:51.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff54000f040 con 0x55733e2be780 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff5400078a0 con 0x55733e2be780 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff557ad9b80 10 client.27395 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.317+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55733e095cd0 con 0x55733e2be780 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff540046080 con 0x55733e2be780 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7ff540020830 con 0x55733e2be780 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 10 client.27395 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 10 client.27395 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 20 client.27395 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e32d000 0x55733e32f420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 4 client.27395 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:51.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55733e05ecf0 con 0x55733e32d000 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.318+0000 7ff548ec0700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e32d000 0x55733e32f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff548ec0700 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e32d000 0x55733e32f420 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e18ba01e254f5c29 server_cookie=88bdb40899f8bbc3 in_seq=0 out_seq=0 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff52cff9700 10 client.27395 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff52cff9700 1 -- 172.21.15.90:0/2705525044 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55733e05ecf0 con 0x55733e32d000 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff52cff9700 10 client.27395 handle_command_reply: tid=0 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 client.27395 shutdown 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 msgr2=0x55733e2cd2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 crypto rx=0x7ff540009610 tx=0x7ff540015ee0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 shutdown_connections 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55733e32d000 0x55733e32f420 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55733e2c8de0 0x55733e2cdd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55733e2be780 0x55733e2cd2c0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 --2- 172.21.15.90:0/2705525044 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55733e2bf0a0 0x55733e2cd800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 >> 172.21.15.90:0/2705525044 conn(0x55733e21d6a0 msgr2=0x55733e2d41e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff52cff9700 0 client.27395 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 shutdown_connections 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 1 -- 172.21.15.90:0/2705525044 wait complete. 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.319+0000 7ff557ad9b80 20 client.27395 trim_cache size 0 max 16384 2024-04-06T14:18:51.325 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 msgr2=0x7ff5300a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 0x7ff5300a3790 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7ff5400062a0 tx=0x7ff5400061f0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 msgr2=0x7ff5441aecb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7ff534009a80 tx=0x7ff534009d90 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 shutdown_connections 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff5300a12d0 0x7ff5300a3790 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff54410f5e0 0x7ff5441b1fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff544109520 0x7ff5441ae770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 --2- 172.21.15.90:0/1161096579 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff544109ee0 0x7ff5441aecb0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 >> 172.21.15.90:0/1161096579 conn(0x7ff544072290 msgr2=0x7ff54406e570 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 shutdown_connections 2024-04-06T14:18:51.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.320+0000 7ff54b925700 1 -- 172.21.15.90:0/1161096579 wait complete. 2024-04-06T14:18:51.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:51 smithi090 ceph-mon[43420]: pgmap v420: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 31 MiB/s wr, 1.92k op/s 2024-04-06T14:18:51.333 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:51 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2316116340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:51.334 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:18:51.335 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:18:51.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:51 smithi134 ceph-mon[47549]: pgmap v420: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 31 MiB/s wr, 1.92k op/s 2024-04-06T14:18:51.403 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:51 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2316116340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:51.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:51 smithi105 ceph-mon[46969]: pgmap v420: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 31 MiB/s wr, 1.92k op/s 2024-04-06T14:18:51.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:51 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2316116340' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 -- 172.21.15.90:0/955899707 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb924100f90 msgr2=0x7fb924101370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 --2- 172.21.15.90:0/955899707 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb924100f90 0x7fb924101370 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 crypto rx=0x7fb91c009f90 tx=0x7fb91c00b330 comp rx=0 tx=0).stop 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 -- 172.21.15.90:0/955899707 shutdown_connections 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 --2- 172.21.15.90:0/955899707 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb92410dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 --2- 172.21.15.90:0/955899707 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb924100f90 0x7fb924101370 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 --2- 172.21.15.90:0/955899707 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924101940 0x7fb924105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 -- 172.21.15.90:0/955899707 >> 172.21.15.90:0/955899707 conn(0x7fb9240fc7f0 msgr2=0x7fb9240fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 -- 172.21.15.90:0/955899707 shutdown_connections 2024-04-06T14:18:51.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.514+0000 7fb92a05b700 1 -- 172.21.15.90:0/955899707 wait complete. 2024-04-06T14:18:51.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 Processor -- start 2024-04-06T14:18:51.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 -- start start 2024-04-06T14:18:51.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924100f90 0x7fb924194fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 0x7fb924199d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb92419a320 con 0x7fb9241060b0 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb92419a490 con 0x7fb9241998a0 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb92a05b700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb92419a650 con 0x7fb924100f90 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36824/0 (socket says 172.21.15.90:36824) 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 -- 172.21.15.90:0/3302442526 learned_addr learned my addr 172.21.15.90:0/3302442526 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb923fff700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 0x7fb924199d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb9237fe700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924100f90 0x7fb924194fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 -- 172.21.15.90:0/3302442526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924100f90 msgr2=0x7fb924194fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924100f90 0x7fb924194fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 -- 172.21.15.90:0/3302442526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 msgr2=0x7fb924199d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 0x7fb924199d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb922ffd700 1 -- 172.21.15.90:0/3302442526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb91c009c40 con 0x7fb9241060b0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.515+0000 7fb923fff700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 0x7fb924199d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb922ffd700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fb90c00e970 tx=0x7fb90c00ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb920ff9700 1 -- 172.21.15.90:0/3302442526 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c009790 con 0x7fb9241060b0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb920ff9700 1 -- 172.21.15.90:0/3302442526 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb90c00eeb0 con 0x7fb9241060b0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb9241a1bb0 con 0x7fb9241060b0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb920ff9700 1 -- 172.21.15.90:0/3302442526 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c0055c0 con 0x7fb9241060b0 2024-04-06T14:18:51.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.516+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9241a2090 con 0x7fb9241060b0 2024-04-06T14:18:51.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.518+0000 7fb920ff9700 1 -- 172.21.15.90:0/3302442526 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb90c004750 con 0x7fb9241060b0 2024-04-06T14:18:51.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.519+0000 7fb920ff9700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 0x7fb9100a7b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.519+0000 7fb920ff9700 1 -- 172.21.15.90:0/3302442526 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb90c014070 con 0x7fb9241060b0 2024-04-06T14:18:51.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.519+0000 7fb9237fe700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 0x7fb9100a7b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.519 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.520+0000 7fb93620fb80 1 Processor -- start 2024-04-06T14:18:51.520 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.520+0000 7fb93620fb80 1 -- start start 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a91185cb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118668a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9117b4990 con 0x55a91185c780 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9117b2350 con 0x55a911866de0 2024-04-06T14:18:51.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb93620fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9117adc40 con 0x55a91185d0a0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb9237fe700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 0x7fb9100a7b60 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fb91c009790 tx=0x7fb91c0096c0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55740/0 (socket says 172.21.15.90:55740) 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 -- 172.21.15.90:0/2305112907 learned_addr learned my addr 172.21.15.90:0/2305112907 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb9237fe700 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a91185cb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 -- 172.21.15.90:0/2305112907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 msgr2=0x55a9118668a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118668a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 -- 172.21.15.90:0/2305112907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 msgr2=0x55a91185cb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a91185cb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.521+0000 7fb923fff700 1 -- 172.21.15.90:0/2305112907 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a91162f3b0 con 0x55a911866de0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.522+0000 7fb923fff700 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fb9140114c0 tx=0x7fb9140117d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=86a781c123d39a74 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.522+0000 7fb919ffb700 1 -- 172.21.15.90:0/2305112907 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb91400e670 con 0x55a911866de0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.522+0000 7fb919ffb700 1 -- 172.21.15.90:0/2305112907 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb91400ec90 con 0x55a911866de0 2024-04-06T14:18:51.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.522+0000 7fb919ffb700 1 -- 172.21.15.90:0/2305112907 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb914019cf0 con 0x55a911866de0 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.522+0000 7fb93620fb80 1 -- 172.21.15.90:0/2305112907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 msgr2=0x55a9118691d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 crypto rx=0x7fb9140114c0 tx=0x7fb9140117d0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 -- 172.21.15.90:0/2305112907 shutdown_connections 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a911866de0 0x55a9118691d0 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a91185cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2305112907 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118668a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 -- 172.21.15.90:0/2305112907 >> 172.21.15.90:0/2305112907 conn(0x55a9117bb6a0 msgr2=0x55a911857a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 -- 172.21.15.90:0/2305112907 shutdown_connections 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 -- 172.21.15.90:0/2305112907 wait complete. 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 -- start start 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.523+0000 7fb93620fb80 1 Processor -- start 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a911864140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a911866de0 0x55a911864680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a911749d80 con 0x55a91185c780 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a911749a20 con 0x55a91185d0a0 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9116316a0 con 0x55a911866de0 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb93620fb80 10 client.0 _ll_register_callbacks cb 0x55a911753470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36848/0 (socket says 172.21.15.90:36848) 2024-04-06T14:18:51.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 -- 172.21.15.90:0/2649160408 learned_addr learned my addr 172.21.15.90:0/2649160408 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb923fff700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a911866de0 0x55a911864680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb922ffd700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a911864140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 -- 172.21.15.90:0/2649160408 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a911866de0 msgr2=0x55a911864680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a911866de0 0x55a911864680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 -- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 msgr2=0x55a911864140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a911864140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 -- 172.21.15.90:0/2649160408 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb914009d10 con 0x55a91185c780 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb9237fe700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 secure :-1 s=READY pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7fb91c015000 tx=0x7fb91c009920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f36b537648bf8f94 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.524+0000 7fb902ffd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb91c005040 con 0x55a91185c780 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb91c022070 con 0x55a91185c780 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a91176e380 con 0x55a91185c780 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb91c0051a0 con 0x55a91185c780 2024-04-06T14:18:51.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 10 client.17682 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a9115dbb60 con 0x55a91185c780 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55a91176e380 con 0x55a91185c780 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb91c0316b0 con 0x55a91185c780 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 10 client.17682 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 10 client.17682 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 20 client.17682 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a911876940 0x55a911878d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 4 client.17682 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.525+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a9117a6700 con 0x55a911876940 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.526+0000 7fb922ffd700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a911876940 0x55a911878d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.526+0000 7fb902ffd700 10 client.17682 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:51.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.526+0000 7fb922ffd700 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a911876940 0x55a911878d60 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c7c07b083c9c05b2 server_cookie=57dff47b00d8fd4a in_seq=0 out_seq=0 2024-04-06T14:18:51.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.527+0000 7fb902ffd700 1 -- 172.21.15.90:0/2649160408 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a9117a6700 con 0x55a911876940 2024-04-06T14:18:51.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.527+0000 7fb902ffd700 10 client.17682 handle_command_reply: tid=0 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.527+0000 7fb93620fb80 1 client.17682 shutdown 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.527+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 msgr2=0x55a911863c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.527+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 secure :-1 s=READY pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7fb91c015000 tx=0x7fb91c009920 comp rx=0 tx=0).stop 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 shutdown_connections 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a911876940 0x55a911878d60 crc :-1 s=READY pgs=227 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a911866de0 0x55a911864680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a91185c780 0x55a911863c00 secure :-1 s=CLOSED pgs=825 cs=0 l=1 rev1=1 crypto rx=0x7fb91c015000 tx=0x7fb91c009920 comp rx=0 tx=0).stop 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 --2- 172.21.15.90:0/2649160408 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a911864140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 >> 172.21.15.90:0/2649160408 conn(0x55a9117bb6a0 msgr2=0x55a911856c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb902ffd700 0 client.17682 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 shutdown_connections 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 1 -- 172.21.15.90:0/2649160408 wait complete. 2024-04-06T14:18:51.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.528+0000 7fb93620fb80 20 client.17682 trim_cache size 0 max 16384 2024-04-06T14:18:51.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 Processor -- start 2024-04-06T14:18:51.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 -- start start 2024-04-06T14:18:51.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118691d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a9118663f0 0x55a911857a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a911858e40 con 0x55a911866de0 2024-04-06T14:18:51.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a911857fd0 con 0x55a9118663f0 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb93620fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a911859220 con 0x55a91185d0a0 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb922ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb922ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36864/0 (socket says 172.21.15.90:36864) 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb922ffd700 1 -- 172.21.15.90:0/368111911 learned_addr learned my addr 172.21.15.90:0/368111911 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb923fff700 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a9118663f0 0x55a911857a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.543+0000 7fb9237fe700 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118691d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 -- 172.21.15.90:0/368111911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 msgr2=0x55a9118691d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118691d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 -- 172.21.15.90:0/368111911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a9118663f0 msgr2=0x55a911857a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a9118663f0 0x55a911857a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 -- 172.21.15.90:0/368111911 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a911773ea0 con 0x55a911866de0 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb922ffd700 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fb90c096540 tx=0x7fb90c005910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6d894c846bd9a467 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb91a7fc700 1 -- 172.21.15.90:0/368111911 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c09de70 con 0x55a911866de0 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb91a7fc700 1 -- 172.21.15.90:0/368111911 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb90c0a24c0 con 0x55a911866de0 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.544+0000 7fb91a7fc700 1 -- 172.21.15.90:0/368111911 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb90c016330 con 0x55a911866de0 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- 172.21.15.90:0/368111911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 msgr2=0x55a911857520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 secure :-1 s=READY pgs=826 cs=0 l=1 rev1=1 crypto rx=0x7fb90c096540 tx=0x7fb90c005910 comp rx=0 tx=0).stop 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- 172.21.15.90:0/368111911 shutdown_connections 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a9118663f0 0x55a911857a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a91185d0a0 0x55a9118691d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 --2- 172.21.15.90:0/368111911 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a911857520 unknown :-1 s=CLOSED pgs=826 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- 172.21.15.90:0/368111911 >> 172.21.15.90:0/368111911 conn(0x55a9117bb6a0 msgr2=0x55a911853890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- 172.21.15.90:0/368111911 shutdown_connections 2024-04-06T14:18:51.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- 172.21.15.90:0/368111911 wait complete. 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 -- start start 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.545+0000 7fb93620fb80 1 Processor -- start 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 0x55a91186c100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a91186c840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a91186cf20 con 0x55a911866de0 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9118d2f90 con 0x55a91185d0a0 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a9118d30d0 con 0x55a9118663f0 2024-04-06T14:18:51.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.546+0000 7fb93620fb80 10 client.0 _ll_register_callbacks cb 0x55a911753470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55762/0 (socket says 172.21.15.90:55762) 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 -- 172.21.15.90:0/1821427834 learned_addr learned my addr 172.21.15.90:0/1821427834 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb922ffd700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 0x55a91186c100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 -- 172.21.15.90:0/1821427834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 msgr2=0x55a91186c100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:51.552 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 0x55a91186c100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 -- 172.21.15.90:0/1821427834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 msgr2=0x55a91186c840 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a91186c840 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb9237fe700 1 -- 172.21.15.90:0/1821427834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9241a2090 con 0x55a91185d0a0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.547+0000 7fb922ffd700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 0x55a91186c100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb9237fe700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fb91c00b370 tx=0x7fb91c031440 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b25543a52e453d9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb91c01b430 con 0x55a91185d0a0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb91c022040 con 0x55a91185d0a0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb91400ec90 con 0x55a91185d0a0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb91c01b590 con 0x55a91185d0a0 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb93620fb80 10 client.27413 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:51.553 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a911633cd0 con 0x55a91185d0a0 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb91400ec90 con 0x55a91185d0a0 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.548+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb91c043860 con 0x55a91185d0a0 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 10 client.27413 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 10 client.27413 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 20 client.27413 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a9118c5980 0x55a9118c7da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 4 client.27413 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55a9115fccf0 con 0x55a9118c5980 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb922ffd700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a9118c5980 0x55a9118c7da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb922ffd700 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a9118c5980 0x55a9118c7da0 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c011ba04175326a9 server_cookie=263d9e1295b3d35f in_seq=0 out_seq=0 2024-04-06T14:18:51.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb902ffd700 10 client.27413 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb902ffd700 1 -- 172.21.15.90:0/1821427834 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55a9115fccf0 con 0x55a9118c5980 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb902ffd700 10 client.27413 handle_command_reply: tid=0 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.549+0000 7fb93620fb80 1 client.27413 shutdown 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 msgr2=0x55a91186dc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7fb91c00b370 tx=0x7fb91c031440 comp rx=0 tx=0).stop 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 shutdown_connections 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a911866de0 0x55a91186c840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a91185d0a0 0x55a91186dc80 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a9118c5980 0x55a9118c7da0 crc :-1 s=READY pgs=228 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 --2- 172.21.15.90:0/1821427834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a9118663f0 0x55a91186c100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 >> 172.21.15.90:0/1821427834 conn(0x55a9117bb6a0 msgr2=0x55a9118574a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb902ffd700 0 client.27413 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:51.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 shutdown_connections 2024-04-06T14:18:51.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 1 -- 172.21.15.90:0/1821427834 wait complete. 2024-04-06T14:18:51.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb93620fb80 20 client.27413 trim_cache size 0 max 16384 2024-04-06T14:18:51.556 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 msgr2=0x7fb9100a7b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 0x7fb9100a7b60 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fb91c009790 tx=0x7fb91c0096c0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 msgr2=0x7fb924195500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 secure :-1 s=READY pgs=824 cs=0 l=1 rev1=1 crypto rx=0x7fb90c00e970 tx=0x7fb90c00ed30 comp rx=0 tx=0).stop 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 shutdown_connections 2024-04-06T14:18:51.557 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9100a56a0 0x7fb9100a7b60 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb9241998a0 0x7fb924199d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb924100f90 0x7fb924194fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 --2- 172.21.15.90:0/3302442526 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb9241060b0 0x7fb924195500 unknown :-1 s=CLOSED pgs=824 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.550+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 >> 172.21.15.90:0/3302442526 conn(0x7fb9240fc7f0 msgr2=0x7fb9240fdbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.551+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 shutdown_connections 2024-04-06T14:18:51.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:51.551+0000 7fb92a05b700 1 -- 172.21.15.90:0/3302442526 wait complete. 2024-04-06T14:18:52.566 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:18:52.566 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:18:52.567 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:18:52.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 -- 172.21.15.90:0/2030895933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107f40 msgr2=0x7fbe2010bf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 --2- 172.21.15.90:0/2030895933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107f40 0x7fbe2010bf80 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7fbe1800b0a0 tx=0x7fbe1800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 -- 172.21.15.90:0/2030895933 shutdown_connections 2024-04-06T14:18:52.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 --2- 172.21.15.90:0/2030895933 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe20114230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 --2- 172.21.15.90:0/2030895933 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe20107590 0x7fbe20107970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 --2- 172.21.15.90:0/2030895933 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107f40 0x7fbe2010bf80 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.790+0000 7fbe28445700 1 -- 172.21.15.90:0/2030895933 >> 172.21.15.90:0/2030895933 conn(0x7fbe200752a0 msgr2=0x7fbe200776c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.791+0000 7fbe28445700 1 -- 172.21.15.90:0/2030895933 shutdown_connections 2024-04-06T14:18:52.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.791+0000 7fbe28445700 1 -- 172.21.15.90:0/2030895933 wait complete. 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.791+0000 7fbe28445700 1 Processor -- start 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.791+0000 7fbe28445700 1 -- start start 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 0x7fbe201a3470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe2019d600 0x7fbe2019da80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe2019e050 con 0x7fbe2019d600 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe2019e190 con 0x7fbe20107590 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe28445700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbe2019e350 con 0x7fbe2010c6b0 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55854/0 (socket says 172.21.15.90:55854) 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 -- 172.21.15.90:0/3219972400 learned_addr learned my addr 172.21.15.90:0/3219972400 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe269e2700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe2019d600 0x7fbe2019da80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.793 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe261e1700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 0x7fbe201a3470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 -- 172.21.15.90:0/3219972400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 msgr2=0x7fbe201a3470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 0x7fbe201a3470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 -- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe2019d600 msgr2=0x7fbe2019da80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe2019d600 0x7fbe2019da80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 -- 172.21.15.90:0/3219972400 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbe18009d00 con 0x7fbe2010c6b0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe261e1700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 0x7fbe201a3470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.792+0000 7fbe259e0700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fbe18007870 tx=0x7fbe180078a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.793+0000 7fbe177fe700 1 -- 172.21.15.90:0/3219972400 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1800e040 con 0x7fbe2010c6b0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.793+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbe201aa330 con 0x7fbe2010c6b0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.793+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe201aa800 con 0x7fbe2010c6b0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.793+0000 7fbe177fe700 1 -- 172.21.15.90:0/3219972400 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbe18004e40 con 0x7fbe2010c6b0 2024-04-06T14:18:52.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.793+0000 7fbe177fe700 1 -- 172.21.15.90:0/3219972400 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe180052a0 con 0x7fbe2010c6b0 2024-04-06T14:18:52.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.797+0000 7fbe345f9b80 1 Processor -- start 2024-04-06T14:18:52.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.797+0000 7fbe345f9b80 1 -- start start 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe177fe700 1 -- 172.21.15.90:0/3219972400 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fbe1801d470 con 0x7fbe2010c6b0 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b4c920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 0x55a494b4f250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494a9aa10 con 0x55a494b43120 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494a983d0 con 0x55a494b42800 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494a93cc0 con 0x55a494b4ce60 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55784/0 (socket says 172.21.15.90:55784) 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 -- 172.21.15.90:0/2449001052 learned_addr learned my addr 172.21.15.90:0/2449001052 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe259e0700 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 0x55a494b4f250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 -- 172.21.15.90:0/2449001052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 msgr2=0x55a494b4f250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 0x55a494b4f250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 -- 172.21.15.90:0/2449001052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 msgr2=0x55a494b4c920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b4c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 -- 172.21.15.90:0/2449001052 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a494915430 con 0x55a494b42800 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe259e0700 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 0x55a494b4f250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe177fe700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 0x7fbe0c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.798+0000 7fbe269e2700 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c009cb0 tx=0x7fbe1c011440 comp rx=0 tx=0).ready entity=mon.2 client_cookie=64cbd791f23db5d7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.799+0000 7fbe177fe700 1 -- 172.21.15.90:0/3219972400 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbe180ce3d0 con 0x7fbe2010c6b0 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.799+0000 7fbe14da5700 1 -- 172.21.15.90:0/2449001052 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c011e30 con 0x55a494b42800 2024-04-06T14:18:52.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.799+0000 7fbe14da5700 1 -- 172.21.15.90:0/2449001052 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbe1c0044f0 con 0x55a494b42800 2024-04-06T14:18:52.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.799+0000 7fbe14da5700 1 -- 172.21.15.90:0/2449001052 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c019420 con 0x55a494b42800 2024-04-06T14:18:52.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.799+0000 7fbe261e1700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 0x7fbe0c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe261e1700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 0x7fbe0c0a3750 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fbe1000bc90 tx=0x7fbe1000b2e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- 172.21.15.90:0/2449001052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 msgr2=0x55a494b42be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c009cb0 tx=0x7fbe1c011440 comp rx=0 tx=0).stop 2024-04-06T14:18:52.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- 172.21.15.90:0/2449001052 shutdown_connections 2024-04-06T14:18:52.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b4ce60 0x55a494b4f250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b42be0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/2449001052 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b4c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- 172.21.15.90:0/2449001052 >> 172.21.15.90:0/2449001052 conn(0x55a494aa1720 msgr2=0x55a494b3db00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- 172.21.15.90:0/2449001052 shutdown_connections 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- 172.21.15.90:0/2449001052 wait complete. 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 -- start start 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.800+0000 7fbe345f9b80 1 Processor -- start 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b4c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494a2fe00 con 0x55a494b43120 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494a2faa0 con 0x55a494b4ce60 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494917720 con 0x55a494b42800 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe345f9b80 10 client.0 _ll_register_callbacks cb 0x55a494a394f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55788/0 (socket says 172.21.15.90:55788) 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 -- 172.21.15.90:0/1472356361 learned_addr learned my addr 172.21.15.90:0/1472356361 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe261e1700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b4c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 -- 172.21.15.90:0/1472356361 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 msgr2=0x55a494b4c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b4c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 -- 172.21.15.90:0/1472356361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 msgr2=0x55a494b49cc0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49cc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 -- 172.21.15.90:0/1472356361 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe1c009960 con 0x55a494b4ce60 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe269e2700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c003c30 tx=0x7fbe1c003960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7661ddffcce5edad server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.801+0000 7fbe09ffb700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.802+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c005590 con 0x55a494b4ce60 2024-04-06T14:18:52.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.802+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbe1c003c90 con 0x55a494b4ce60 2024-04-06T14:18:52.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.802+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55a494a54400 con 0x55a494b4ce60 2024-04-06T14:18:52.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbe1c0056f0 con 0x55a494b4ce60 2024-04-06T14:18:52.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 10 client.27425 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:52.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a4948c1be0 con 0x55a494b4ce60 2024-04-06T14:18:52.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbe1c030030 con 0x55a494b4ce60 2024-04-06T14:18:52.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbe1c021040 con 0x55a494b4ce60 2024-04-06T14:18:52.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 10 client.27425 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 10 client.27425 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 20 client.27425 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494b5ca20 0x55a494b5ee40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 4 client.27425 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.804+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55a494a8c780 con 0x55a494b5ca20 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.805+0000 7fbe261e1700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494b5ca20 0x55a494b5ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.805+0000 7fbe261e1700 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494b5ca20 0x55a494b5ee40 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=68cd8b9df9dde96b server_cookie=7d45aece5e99fbef in_seq=0 out_seq=0 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.805+0000 7fbe09ffb700 10 client.27425 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe09ffb700 1 -- 172.21.15.90:0/1472356361 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55a494a8c780 con 0x55a494b5ca20 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe09ffb700 10 client.27425 handle_command_reply: tid=0 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 client.27425 shutdown 2024-04-06T14:18:52.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 msgr2=0x55a494b4a400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c003c30 tx=0x7fbe1c003960 comp rx=0 tx=0).stop 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 shutdown_connections 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b4a400 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494b5ca20 0x55a494b5ee40 crc :-1 s=READY pgs=229 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b4c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1472356361 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 >> 172.21.15.90:0/1472356361 conn(0x55a494aa1720 msgr2=0x55a494b3cca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe09ffb700 0 client.27425 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 shutdown_connections 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1472356361 wait complete. 2024-04-06T14:18:52.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.806+0000 7fbe345f9b80 20 client.27425 trim_cache size 0 max 16384 2024-04-06T14:18:52.841 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 Processor -- start 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 -- start start 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b3dcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b46f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494b43500 con 0x55a494b43120 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494b4a9e0 con 0x55a494b4ce60 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.840+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494b43a90 con 0x55a494b42800 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.842+0000 7fbe259e0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.842 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.842+0000 7fbe259e0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36912/0 (socket says 172.21.15.90:36912) 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.842+0000 7fbe259e0700 1 -- 172.21.15.90:0/1588281283 learned_addr learned my addr 172.21.15.90:0/1588281283 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 -- 172.21.15.90:0/1588281283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 msgr2=0x55a494b3dcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b3dcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 -- 172.21.15.90:0/1588281283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 msgr2=0x55a494b46f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b46f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.843 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 -- 172.21.15.90:0/1588281283 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55a494a59f20 con 0x55a494b43120 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe259e0700 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fbe180062a0 tx=0x7fbe18097890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=467a1c85dbe5ddfa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.843+0000 7fbe155a6700 1 -- 172.21.15.90:0/1588281283 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1809bc50 con 0x55a494b43120 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.844+0000 7fbe155a6700 1 -- 172.21.15.90:0/1588281283 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbe1800be20 con 0x55a494b43120 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.844+0000 7fbe155a6700 1 -- 172.21.15.90:0/1588281283 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe180a39d0 con 0x55a494b43120 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1588281283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 msgr2=0x55a494b49580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 secure :-1 s=READY pgs=827 cs=0 l=1 rev1=1 crypto rx=0x7fbe180062a0 tx=0x7fbe18097890 comp rx=0 tx=0).stop 2024-04-06T14:18:52.844 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1588281283 shutdown_connections 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b4ce60 0x55a494b46f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b42800 0x55a494b3dcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/1588281283 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b43120 0x55a494b49580 unknown :-1 s=CLOSED pgs=827 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1588281283 >> 172.21.15.90:0/1588281283 conn(0x55a494aa1720 msgr2=0x55a494b3d5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1588281283 shutdown_connections 2024-04-06T14:18:52.845 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- 172.21.15.90:0/1588281283 wait complete. 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.845+0000 7fbe345f9b80 1 -- start start 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 Processor -- start 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b52dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b43120 0x55a494b53310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494bbad70 con 0x55a494b4ce60 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494bbaee0 con 0x55a494b42800 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55a494bbb0a0 con 0x55a494b43120 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe345f9b80 10 client.0 _ll_register_callbacks cb 0x55a494a394f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe269e2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36922/0 (socket says 172.21.15.90:36922) 2024-04-06T14:18:52.846 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.846+0000 7fbe269e2700 1 -- 172.21.15.90:0/142475040 learned_addr learned my addr 172.21.15.90:0/142475040 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:52.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 -- 172.21.15.90:0/142475040 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b43120 msgr2=0x55a494b53310 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:52.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b43120 0x55a494b53310 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 -- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 msgr2=0x55a494b52dd0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b52dd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 -- 172.21.15.90:0/142475040 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbe18097040 con 0x55a494b4ce60 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe269e2700 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c024c40 tx=0x7fbe1c004f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=658272771849abc4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe09ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbe1c004b20 con 0x55a494b4ce60 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.847+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbe1c023720 con 0x55a494b4ce60 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.848+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbe1c0044f0 con 0x55a494b4ce60 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.848+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbe1c009280 con 0x55a494b4ce60 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.848+0000 7fbe345f9b80 10 client.17697 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:52.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.848+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55a494919d50 con 0x55a494b4ce60 2024-04-06T14:18:52.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.848+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbe1c0044f0 con 0x55a494b4ce60 2024-04-06T14:18:52.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fbe1c011560 con 0x55a494b4ce60 2024-04-06T14:18:52.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 10 client.17697 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 10 client.17697 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 20 client.17697 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494bc3760 0x55a494bc5b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 4 client.17697 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.849+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55a4948e2d70 con 0x55a494bc3760 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.850+0000 7fbe261e1700 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494bc3760 0x55a494bc5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.850+0000 7fbe09ffb700 10 client.17697 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:52.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.850+0000 7fbe261e1700 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494bc3760 0x55a494bc5b80 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3ef73d2737c3a7ce server_cookie=2fbd230e159d0050 in_seq=0 out_seq=0 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe09ffb700 1 -- 172.21.15.90:0/142475040 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55a4948e2d70 con 0x55a494bc3760 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe09ffb700 10 client.17697 handle_command_reply: tid=0 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 client.17697 shutdown 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 msgr2=0x55a494bba830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 secure :-1 s=READY pgs=828 cs=0 l=1 rev1=1 crypto rx=0x7fbe1c024c40 tx=0x7fbe1c004f90 comp rx=0 tx=0).stop 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 shutdown_connections 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55a494b4ce60 0x55a494bba830 unknown :-1 s=CLOSED pgs=828 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55a494b42800 0x55a494b52dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55a494bc3760 0x55a494bc5b80 crc :-1 s=READY pgs=230 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.854 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 --2- 172.21.15.90:0/142475040 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55a494b43120 0x55a494b53310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe09ffb700 0 client.17697 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:52.855 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.854+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 >> 172.21.15.90:0/142475040 conn(0x55a494aa1720 msgr2=0x55a494b4b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.859+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 shutdown_connections 2024-04-06T14:18:52.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.859+0000 7fbe345f9b80 1 -- 172.21.15.90:0/142475040 wait complete. 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe345f9b80 20 client.17697 trim_cache size 0 max 16384 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "11a8c2a9-5ff3-4e66-9905-aae1c46b5bfd", 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:18:52.859 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:52.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 msgr2=0x7fbe0c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 0x7fbe0c0a3750 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fbe1000bc90 tx=0x7fbe1000b2e0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 msgr2=0x7fbe201a39b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:52.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fbe18007870 tx=0x7fbe180078a0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.860 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 shutdown_connections 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbe0c0a1290 0x7fbe0c0a3750 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbe2019d600 0x7fbe2019da80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbe20107590 0x7fbe201a3470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 --2- 172.21.15.90:0/3219972400 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbe2010c6b0 0x7fbe201a39b0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.860+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 >> 172.21.15.90:0/3219972400 conn(0x7fbe200752a0 msgr2=0x7fbe20076610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.861+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 shutdown_connections 2024-04-06T14:18:52.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:52.861+0000 7fbe28445700 1 -- 172.21.15.90:0/3219972400 wait complete. 2024-04-06T14:18:52.877 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:18:52.878 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:18:53.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 785.0 s, 1204.6 tps, lat 83.255 ms stddev 99.291 2024-04-06T14:18:53.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.094+0000 7f1d1ff00700 1 -- 172.21.15.90:0/1204432897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 msgr2=0x7f1d18103960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.094+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/1204432897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18103960 secure :-1 s=READY pgs=829 cs=0 l=1 rev1=1 crypto rx=0x7f1d1000b0a0 tx=0x7f1d1000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.101+0000 7f1d1ff00700 1 -- 172.21.15.90:0/1204432897 shutdown_connections 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.101+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/1204432897 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18103960 unknown :-1 s=CLOSED pgs=829 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.101+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/1204432897 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d18105e60 0x7f1d18106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.101+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/1204432897 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 0x7f1d180ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.101+0000 7f1d1ff00700 1 -- 172.21.15.90:0/1204432897 >> 172.21.15.90:0/1204432897 conn(0x7f1d180761b0 msgr2=0x7f1d180fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.103+0000 7f1d1ff00700 1 -- 172.21.15.90:0/1204432897 shutdown_connections 2024-04-06T14:18:53.103 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.104+0000 7f1d1ff00700 1 -- 172.21.15.90:0/1204432897 wait complete. 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.105+0000 7f1d1ff00700 1 Processor -- start 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.105+0000 7f1d1ff00700 1 -- start start 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 0x7f1d180723c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d18105e60 0x7f1d1806b5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d1806bd80 con 0x7f1d180ffcc0 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d1806bef0 con 0x7f1d180ff220 2024-04-06T14:18:53.106 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1ff00700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d1806c0b0 con 0x7f1d18105e60 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36944/0 (socket says 172.21.15.90:36944) 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 -- 172.21.15.90:0/3984341064 learned_addr learned my addr 172.21.15.90:0/3984341064 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 0x7f1d180723c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 -- 172.21.15.90:0/3984341064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d18105e60 msgr2=0x7f1d1806b5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d18105e60 0x7f1d1806b5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.108 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 -- 172.21.15.90:0/3984341064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 msgr2=0x7f1d180723c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 0x7f1d180723c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.106+0000 7f1d1d49b700 1 -- 172.21.15.90:0/3984341064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d10009d00 con 0x7f1d180ffcc0 2024-04-06T14:18:53.109 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.107+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f1d0800c920 tx=0x7f1d0800cc30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.110+0000 7f1d0effd700 1 -- 172.21.15.90:0/3984341064 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d08007b00 con 0x7f1d180ffcc0 2024-04-06T14:18:53.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.110+0000 7f1d0effd700 1 -- 172.21.15.90:0/3984341064 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d0800ce70 con 0x7f1d180ffcc0 2024-04-06T14:18:53.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.110+0000 7f1d0effd700 1 -- 172.21.15.90:0/3984341064 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d080054b0 con 0x7f1d180ffcc0 2024-04-06T14:18:53.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.110+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d1806ff50 con 0x7f1d180ffcc0 2024-04-06T14:18:53.110 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.110+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d18070410 con 0x7f1d180ffcc0 2024-04-06T14:18:53.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.115+0000 7f1d2c0b4b80 1 Processor -- start 2024-04-06T14:18:53.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 -- start start 2024-04-06T14:18:53.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d0effd700 1 -- 172.21.15.90:0/3984341064 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f1d08004750 con 0x7f1d180ffcc0 2024-04-06T14:18:53.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f19b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f238a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f261d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650e71990 con 0x564650f1a0a0 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650e6f350 con 0x564650f19780 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650e6ac40 con 0x564650f23de0 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d1dc9c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f238a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d1dc9c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f238a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36952/0 (socket says 172.21.15.90:36952) 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d1dc9c700 1 -- 172.21.15.90:0/1682047485 learned_addr learned my addr 172.21.15.90:0/1682047485 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.116+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f19b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 -- 172.21.15.90:0/1682047485 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 msgr2=0x564650f261d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f261d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f261d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 -- 172.21.15.90:0/1682047485 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 msgr2=0x564650f238a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f238a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 -- 172.21.15.90:0/1682047485 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564650cec3b0 con 0x564650f19780 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f19b60 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f1d1000b070 tx=0x7f1d10005ae0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=27b7dacf101dd8e0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d03fff700 1 -- 172.21.15.90:0/1682047485 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d1000e040 con 0x564650f19780 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d03fff700 1 -- 172.21.15.90:0/1682047485 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d10009490 con 0x564650f19780 2024-04-06T14:18:53.117 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d03fff700 1 -- 172.21.15.90:0/1682047485 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d10005290 con 0x564650f19780 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.117+0000 7f1d0effd700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 0x7f1d040a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d0effd700 1 -- 172.21.15.90:0/3984341064 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d080cc890 con 0x7f1d180ffcc0 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1682047485 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 msgr2=0x564650f19b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f19b60 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f1d1000b070 tx=0x7f1d10005ae0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1682047485 shutdown_connections 2024-04-06T14:18:53.118 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f261d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f19b60 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.118+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1682047485 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f238a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.119+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1682047485 >> 172.21.15.90:0/1682047485 conn(0x564650e786a0 msgr2=0x564650f14a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1682047485 shutdown_connections 2024-04-06T14:18:53.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 0x7f1d040a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1682047485 wait complete. 2024-04-06T14:18:53.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d2c0b4b80 1 -- start start 2024-04-06T14:18:53.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 0x7f1d040a3920 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f1d14005cb0 tx=0x7f1d1400a300 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.122+0000 7f1d2c0b4b80 1 Processor -- start 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f20b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f21110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650e06d80 con 0x564650f1a0a0 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650e06a20 con 0x564650f19780 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650cee6a0 con 0x564650f23de0 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d2c0b4b80 10 client.0 _ll_register_callbacks cb 0x564650e10470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1dc9c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1dc9c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:55866/0 (socket says 172.21.15.90:55866) 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1dc9c700 1 -- 172.21.15.90:0/1254032437 learned_addr learned my addr 172.21.15.90:0/1254032437 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:53.123 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f21110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f20b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.123+0000 7f1d1dc9c700 1 -- 172.21.15.90:0/1254032437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 msgr2=0x564650f21110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f21110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1dc9c700 1 -- 172.21.15.90:0/1254032437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 msgr2=0x564650f20b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f20b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1dc9c700 1 -- 172.21.15.90:0/1254032437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d18104490 con 0x564650f19780 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f20b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f1d140109e0 tx=0x7f1d14004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8aff70f0e4a61e6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d017fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d1401d890 con 0x564650f19780 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d1401d9f0 con 0x564650f19780 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.124+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564650e2b380 con 0x564650f19780 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.125+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d1401dbb0 con 0x564650f19780 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.125+0000 7f1d2c0b4b80 10 client.27434 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:53.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.125+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564650c98b60 con 0x564650f19780 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564650e2b380 con 0x564650f19780 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1d1401a3c0 con 0x564650f19780 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 10 client.27434 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 10 client.27434 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 20 client.27434 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f33970 0x564650f35d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 4 client.27434 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:18:53.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.126+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564650e63700 con 0x564650f33970 2024-04-06T14:18:53.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.128+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f33970 0x564650f35d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.128+0000 7f1d017fa700 10 client.27434 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:53.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.128+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f33970 0x564650f35d90 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3e3dacca0d40552e server_cookie=5eac26def5a8c12b in_seq=0 out_seq=0 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d017fa700 1 -- 172.21.15.90:0/1254032437 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x564650e63700 con 0x564650f33970 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d017fa700 10 client.27434 handle_command_reply: tid=0 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 client.27434 shutdown 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 msgr2=0x564650f237e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f1d140109e0 tx=0x7f1d14004750 comp rx=0 tx=0).stop 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 shutdown_connections 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f33970 0x564650f35d90 crc :-1 s=READY pgs=231 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f23de0 0x564650f21110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f237e0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.129 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/1254032437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f1a0a0 0x564650f20b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 >> 172.21.15.90:0/1254032437 conn(0x564650e786a0 msgr2=0x564650f13c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d017fa700 0 client.27434 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:53.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 shutdown_connections 2024-04-06T14:18:53.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.129+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/1254032437 wait complete. 2024-04-06T14:18:53.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.130+0000 7f1d2c0b4b80 20 client.27434 trim_cache size 0 max 16384 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 Processor -- start 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 -- start start 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f210b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f20ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f22080 con 0x564650f23de0 2024-04-06T14:18:53.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f216f0 con 0x564650f19780 2024-04-06T14:18:53.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.148+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f2af40 con 0x564650f1a0a0 2024-04-06T14:18:53.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36976/0 (socket says 172.21.15.90:36976) 2024-04-06T14:18:53.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 -- 172.21.15.90:0/51396154 learned_addr learned my addr 172.21.15.90:0/51396154 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 -- 172.21.15.90:0/51396154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 msgr2=0x564650f20ac0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f20ac0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 -- 172.21.15.90:0/51396154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 msgr2=0x564650f210b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f210b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 -- 172.21.15.90:0/51396154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564650e30ea0 con 0x564650f23de0 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.149+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7f1d1000b070 tx=0x7f1d10007b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6c30d033c0a8409b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d0cff9700 1 -- 172.21.15.90:0/51396154 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d1000e040 con 0x564650f23de0 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d0cff9700 1 -- 172.21.15.90:0/51396154 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d10003a00 con 0x564650f23de0 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d0cff9700 1 -- 172.21.15.90:0/51396154 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d100056b0 con 0x564650f23de0 2024-04-06T14:18:53.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/51396154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 msgr2=0x564650f2aa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 secure :-1 s=READY pgs=831 cs=0 l=1 rev1=1 crypto rx=0x7f1d1000b070 tx=0x7f1d10007b70 comp rx=0 tx=0).stop 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/51396154 shutdown_connections 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f2aa00 unknown :-1 s=CLOSED pgs=831 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f210b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.150+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/51396154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f20ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.151+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/51396154 >> 172.21.15.90:0/51396154 conn(0x564650e786a0 msgr2=0x564650f20390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.151+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/51396154 shutdown_connections 2024-04-06T14:18:53.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.151+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/51396154 wait complete. 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.151+0000 7f1d2c0b4b80 1 -- start start 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.151+0000 7f1d2c0b4b80 1 Processor -- start 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f28bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f29110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f7fcd0 con 0x564650f23de0 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f7fe40 con 0x564650f19780 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x564650f80000 con 0x564650f1a0a0 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d2c0b4b80 10 client.0 _ll_register_callbacks cb 0x564650e10470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d1e49d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d1e49d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:36982/0 (socket says 172.21.15.90:36982) 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d1e49d700 1 -- 172.21.15.90:0/2081177113 learned_addr learned my addr 172.21.15.90:0/2081177113 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:18:53.155 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.152+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f28bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1d49b700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f29110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 -- 172.21.15.90:0/2081177113 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 msgr2=0x564650f29110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f29110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 -- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 msgr2=0x564650f28bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f28bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 -- 172.21.15.90:0/2081177113 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564650cec3b0 con 0x564650f23de0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d1e49d700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f1d10000f80 tx=0x7f1d100076d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=38651ef08c10c02a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d017fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d1000e040 con 0x564650f23de0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(98..98 src has 1..98) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1d100054c0 con 0x564650f23de0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1d10009490 con 0x564650f23de0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d100056c0 con 0x564650f23de0 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d2c0b4b80 10 client.17715 fetch_fsmap learned FSMap version 24 2024-04-06T14:18:53.156 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.153+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x564650cf0cd0 con 0x564650f23de0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d1003c080 con 0x564650f23de0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1d1002e070 con 0x564650f23de0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 10 client.17715 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 10 client.17715 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 20 client.17715 populate_metadata read hostname 'smithi090' 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f88520 0x564650f8a910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 4 client.17715 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x564650cb9cf0 con 0x564650f88520 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f88520 0x564650f8a910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d017fa700 10 client.17715 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.154+0000 7f1d1dc9c700 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f88520 0x564650f8a910 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a0202c3b3ff1ab7e server_cookie=ebaaca0384c1f362 in_seq=0 out_seq=0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d017fa700 1 -- 172.21.15.90:0/2081177113 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x564650cb9cf0 con 0x564650f88520 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d017fa700 10 client.17715 handle_command_reply: tid=0 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 client.17715 shutdown 2024-04-06T14:18:53.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 msgr2=0x564650f7f720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 secure :-1 s=READY pgs=832 cs=0 l=1 rev1=1 crypto rx=0x7f1d10000f80 tx=0x7f1d100076d0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 shutdown_connections 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x564650f23de0 0x564650f7f720 unknown :-1 s=CLOSED pgs=832 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x564650f19780 0x564650f28bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x564650f88520 0x564650f8a910 crc :-1 s=READY pgs=232 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 --2- 172.21.15.90:0/2081177113 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x564650f1a0a0 0x564650f29110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 >> 172.21.15.90:0/2081177113 conn(0x564650e786a0 msgr2=0x564650f1f3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d017fa700 0 client.17715 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 shutdown_connections 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 1 -- 172.21.15.90:0/2081177113 wait complete. 2024-04-06T14:18:53.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.155+0000 7f1d2c0b4b80 20 client.17715 trim_cache size 0 max 16384 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "IDLE (waiting for more scrubs)", 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "11a8c2a9-5ff3-4e66-9905-aae1c46b5bfd": { 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "11a8c2a9-5ff3-4e66-9905-aae1c46b5bfd", 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:18:53.159 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 msgr2=0x7f1d040a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 0x7f1d040a3920 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f1d14005cb0 tx=0x7f1d1400a300 comp rx=0 tx=0).stop 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 msgr2=0x7f1d18072940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 secure :-1 s=READY pgs=830 cs=0 l=1 rev1=1 crypto rx=0x7f1d0800c920 tx=0x7f1d0800cc30 comp rx=0 tx=0).stop 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 shutdown_connections 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f1d040a1460 0x7f1d040a3920 secure :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f1d14005cb0 tx=0x7f1d1400a300 comp rx=0 tx=0).stop 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f1d18105e60 0x7f1d1806b5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d180ff220 0x7f1d180723c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 --2- 172.21.15.90:0/3984341064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f1d180ffcc0 0x7f1d18072940 unknown :-1 s=CLOSED pgs=830 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:18:53.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.156+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 >> 172.21.15.90:0/3984341064 conn(0x7f1d180761b0 msgr2=0x7f1d181014c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:18:53.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.157+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 shutdown_connections 2024-04-06T14:18:53.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:18:53.157+0000 7f1d1ff00700 1 -- 172.21.15.90:0/3984341064 wait complete. 2024-04-06T14:18:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:53 smithi090 ceph-mon[43420]: pgmap v421: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 36 MiB/s wr, 2.24k op/s 2024-04-06T14:18:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:53 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:18:53.173 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:53 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:53.174 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:53 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:18:53.174 INFO:tasks.cephfs.filesystem:scrub status for tag:11a8c2a9-5ff3-4e66-9905-aae1c46b5bfd - {'path': '/', 'tag': '11a8c2a9-5ff3-4e66-9905-aae1c46b5bfd', 'options': 'recursive,force'} 2024-04-06T14:18:53.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:53 smithi134 ceph-mon[47549]: pgmap v421: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 36 MiB/s wr, 2.24k op/s 2024-04-06T14:18:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:53 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:18:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:53 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:53.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:53 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:18:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:53 smithi105 ceph-mon[46969]: pgmap v421: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 36 MiB/s wr, 2.24k op/s 2024-04-06T14:18:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:53 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:18:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:53 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:53 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:18:54.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:54 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:54.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:54 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:54.542 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:54 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:18:55.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:55 smithi134 ceph-mon[47549]: pgmap v422: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 27 MiB/s wr, 1.74k op/s 2024-04-06T14:18:55.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:55 smithi105 ceph-mon[46969]: pgmap v422: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 27 MiB/s wr, 1.74k op/s 2024-04-06T14:18:55.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:55 smithi090 ceph-mon[43420]: pgmap v422: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 27 MiB/s wr, 1.74k op/s 2024-04-06T14:18:56.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:56 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:18:56.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:56 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:18:56.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:56 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:57 smithi105 ceph-mon[46969]: pgmap v423: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:18:57.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:57 smithi090 ceph-mon[43420]: pgmap v423: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:18:57.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:57 smithi134 ceph-mon[47549]: pgmap v423: 97 pgs: 97 active+clean; 6.2 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:18:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 790.0 s, 1445.8 tps, lat 68.341 ms stddev 77.297 2024-04-06T14:18:59.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:18:59 smithi090 ceph-mon[43420]: pgmap v424: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 26 MiB/s wr, 1.66k op/s 2024-04-06T14:18:59.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:18:59 smithi134 ceph-mon[47549]: pgmap v424: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 26 MiB/s wr, 1.66k op/s 2024-04-06T14:18:59.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:18:59 smithi105 ceph-mon[46969]: pgmap v424: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 26 MiB/s wr, 1.66k op/s 2024-04-06T14:19:01.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:01 smithi105 ceph-mon[46969]: pgmap v425: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 19 MiB/s wr, 1.35k op/s 2024-04-06T14:19:01.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:01 smithi090 ceph-mon[43420]: pgmap v425: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 19 MiB/s wr, 1.35k op/s 2024-04-06T14:19:01.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:01 smithi134 ceph-mon[47549]: pgmap v425: 97 pgs: 97 active+clean; 6.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 19 MiB/s wr, 1.35k op/s 2024-04-06T14:19:03.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 795.0 s, 1064.4 tps, lat 94.806 ms stddev 146.369 2024-04-06T14:19:03.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:03 smithi105 ceph-mon[46969]: pgmap v426: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 223 KiB/s rd, 22 MiB/s wr, 1.57k op/s 2024-04-06T14:19:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:03 smithi090 ceph-mon[43420]: pgmap v426: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 223 KiB/s rd, 22 MiB/s wr, 1.57k op/s 2024-04-06T14:19:03.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:03 smithi134 ceph-mon[47549]: pgmap v426: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 223 KiB/s rd, 22 MiB/s wr, 1.57k op/s 2024-04-06T14:19:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:05 smithi105 ceph-mon[46969]: pgmap v427: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 374 KiB/s rd, 20 MiB/s wr, 1.47k op/s 2024-04-06T14:19:05.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:05 smithi090 ceph-mon[43420]: pgmap v427: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 374 KiB/s rd, 20 MiB/s wr, 1.47k op/s 2024-04-06T14:19:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:05 smithi134 ceph-mon[47549]: pgmap v427: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 374 KiB/s rd, 20 MiB/s wr, 1.47k op/s 2024-04-06T14:19:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:07 smithi090 ceph-mon[43420]: pgmap v428: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 366 KiB/s rd, 17 MiB/s wr, 1.25k op/s 2024-04-06T14:19:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:07 smithi090 ceph-mon[43420]: osdmap e99: 12 total, 12 up, 12 in 2024-04-06T14:19:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:07 smithi134 ceph-mon[47549]: pgmap v428: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 366 KiB/s rd, 17 MiB/s wr, 1.25k op/s 2024-04-06T14:19:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:07 smithi134 ceph-mon[47549]: osdmap e99: 12 total, 12 up, 12 in 2024-04-06T14:19:07.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:07 smithi105 ceph-mon[46969]: pgmap v428: 97 pgs: 97 active+clean; 6.3 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 366 KiB/s rd, 17 MiB/s wr, 1.25k op/s 2024-04-06T14:19:07.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:07 smithi105 ceph-mon[46969]: osdmap e99: 12 total, 12 up, 12 in 2024-04-06T14:19:08.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 800.0 s, 1113.2 tps, lat 90.068 ms stddev 140.587 2024-04-06T14:19:09.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:09 smithi090 ceph-mon[43420]: pgmap v430: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 16 MiB/s wr, 1.17k op/s 2024-04-06T14:19:09.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:09 smithi134 ceph-mon[47549]: pgmap v430: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 16 MiB/s wr, 1.17k op/s 2024-04-06T14:19:09.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:09 smithi105 ceph-mon[46969]: pgmap v430: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 16 MiB/s wr, 1.17k op/s 2024-04-06T14:19:11.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:11 smithi134 ceph-mon[47549]: pgmap v431: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 15 MiB/s wr, 1.09k op/s 2024-04-06T14:19:11.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:11 smithi105 ceph-mon[46969]: pgmap v431: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 15 MiB/s wr, 1.09k op/s 2024-04-06T14:19:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:11 smithi090 ceph-mon[43420]: pgmap v431: 97 pgs: 13 active+clean+snaptrim, 21 active+clean+snaptrim_wait, 63 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 443 KiB/s rd, 15 MiB/s wr, 1.09k op/s 2024-04-06T14:19:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 805.0 s, 1515.4 tps, lat 66.199 ms stddev 71.228 2024-04-06T14:19:13.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:13 smithi105 ceph-mon[46969]: pgmap v432: 97 pgs: 3 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 88 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 216 KiB/s rd, 12 MiB/s wr, 825 op/s 2024-04-06T14:19:13.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:13 smithi090 ceph-mon[43420]: pgmap v432: 97 pgs: 3 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 88 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 216 KiB/s rd, 12 MiB/s wr, 825 op/s 2024-04-06T14:19:13.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:13 smithi134 ceph-mon[47549]: pgmap v432: 97 pgs: 3 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 88 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 216 KiB/s rd, 12 MiB/s wr, 825 op/s 2024-04-06T14:19:15.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:15 smithi134 ceph-mon[47549]: pgmap v433: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 729 op/s 2024-04-06T14:19:15.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:15 smithi105 ceph-mon[46969]: pgmap v433: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 729 op/s 2024-04-06T14:19:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:15 smithi090 ceph-mon[43420]: pgmap v433: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 729 op/s 2024-04-06T14:19:17.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:17 smithi105 ceph-mon[46969]: pgmap v434: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 724 op/s 2024-04-06T14:19:17.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:17 smithi090 ceph-mon[43420]: pgmap v434: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 724 op/s 2024-04-06T14:19:17.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:17 smithi134 ceph-mon[47549]: pgmap v434: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 61 KiB/s rd, 12 MiB/s wr, 724 op/s 2024-04-06T14:19:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 810.0 s, 1544.4 tps, lat 64.538 ms stddev 73.931 2024-04-06T14:19:19.297 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:18 smithi090 ceph-mon[43420]: osdmap e100: 12 total, 12 up, 12 in 2024-04-06T14:19:19.297 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:18 smithi090 ceph-mon[43420]: pgmap v436: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 12 MiB/s wr, 630 op/s 2024-04-06T14:19:19.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:19 smithi134 ceph-mon[47549]: osdmap e100: 12 total, 12 up, 12 in 2024-04-06T14:19:19.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:19 smithi134 ceph-mon[47549]: pgmap v436: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 12 MiB/s wr, 630 op/s 2024-04-06T14:19:19.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:19 smithi105 ceph-mon[46969]: osdmap e100: 12 total, 12 up, 12 in 2024-04-06T14:19:19.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:19 smithi105 ceph-mon[46969]: pgmap v436: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 12 MiB/s wr, 630 op/s 2024-04-06T14:19:21.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:21 smithi105 ceph-mon[46969]: pgmap v437: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 623 op/s 2024-04-06T14:19:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:21 smithi090 ceph-mon[43420]: pgmap v437: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 623 op/s 2024-04-06T14:19:21.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:21 smithi134 ceph-mon[47549]: pgmap v437: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 623 op/s 2024-04-06T14:19:23.024 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 815.0 s, 1495.6 tps, lat 67.140 ms stddev 73.373 2024-04-06T14:19:23.176 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:19:23.176 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:19:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.450+0000 7f6ed7828700 1 -- 172.21.15.90:0/23929101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed01060b0 msgr2=0x7f6ed010dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.450+0000 7f6ed7828700 1 --2- 172.21.15.90:0/23929101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed01060b0 0x7f6ed010dc30 secure :-1 s=READY pgs=833 cs=0 l=1 rev1=1 crypto rx=0x7f6ec800b0a0 tx=0x7f6ec800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 -- 172.21.15.90:0/23929101 shutdown_connections 2024-04-06T14:19:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 --2- 172.21.15.90:0/23929101 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed01060b0 0x7f6ed010dc30 unknown :-1 s=CLOSED pgs=833 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 --2- 172.21.15.90:0/23929101 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed00691d0 0x7f6ed00695b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 --2- 172.21.15.90:0/23929101 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed0069b80 0x7f6ed0105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 -- 172.21.15.90:0/23929101 >> 172.21.15.90:0/23929101 conn(0x7f6ed00769b0 msgr2=0x7f6ed0076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 -- 172.21.15.90:0/23929101 shutdown_connections 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 -- 172.21.15.90:0/23929101 wait complete. 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 Processor -- start 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.451+0000 7f6ed7828700 1 -- start start 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 0x7f6ed019f230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed01060b0 0x7f6ed019f770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed0199e60 con 0x7f6ed0199440 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed0199fd0 con 0x7f6ed01060b0 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed7828700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ed019a190 con 0x7f6ed00691d0 2024-04-06T14:19:23.454 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed55c4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 0x7f6ed019f230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54960/0 (socket says 172.21.15.90:54960) 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed55c4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 0x7f6ed019f230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48554/0 (socket says 172.21.15.90:48554) 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/1153865648 learned_addr learned my addr 172.21.15.90:0/1153865648 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed01060b0 0x7f6ed019f770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/1153865648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 msgr2=0x7f6ed019f230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 0x7f6ed019f230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/1153865648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed01060b0 msgr2=0x7f6ed019f770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed01060b0 0x7f6ed019f770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/1153865648 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6ec8009d00 con 0x7f6ed0199440 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.452+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f6ec800b070 tx=0x7f6ec8007cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.453+0000 7f6ec67fc700 1 -- 172.21.15.90:0/1153865648 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec800e040 con 0x7f6ed0199440 2024-04-06T14:19:23.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.453+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6ed01a6040 con 0x7f6ed0199440 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.453+0000 7f6ec67fc700 1 -- 172.21.15.90:0/1153865648 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ec800bdc0 con 0x7f6ed0199440 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.453+0000 7f6ec67fc700 1 -- 172.21.15.90:0/1153865648 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec80052a0 con 0x7f6ed0199440 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.453+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ed01a6410 con 0x7f6ed0199440 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.455+0000 7f6ec67fc700 1 -- 172.21.15.90:0/1153865648 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f6ec8005400 con 0x7f6ed0199440 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.456+0000 7f6ec67fc700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 0x7f6ebc0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.456 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.456+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 0x7f6ebc0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.456+0000 7f6ec67fc700 1 -- 172.21.15.90:0/1153865648 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6ec80cdd50 con 0x7f6ed0199440 2024-04-06T14:19:23.457 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 Processor -- start 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 -- start start 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 0x7f6ebc0a3750 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6ecc009c60 tx=0x7f6ecc009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90b9370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b98b0 0x55ffa90c30b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9008990 con 0x55ffa90b98b0 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9006350 con 0x55ffa90c35f0 2024-04-06T14:19:23.458 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.457+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9001c40 con 0x55ffa90b8f90 2024-04-06T14:19:23.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.459 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:36962/0 (socket says 172.21.15.90:36962) 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/892979095 learned_addr learned my addr 172.21.15.90:0/892979095 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90b9370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/892979095 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 msgr2=0x55ffa90b9370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90b9370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/892979095 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b98b0 msgr2=0x55ffa90c30b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b98b0 0x55ffa90c30b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/892979095 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ffa8e833b0 con 0x55ffa90c35f0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.458+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f6ec800cfe0 tx=0x7f6ec80099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=851f7a7216c40f0c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.459+0000 7f6ebb7fe700 1 -- 172.21.15.90:0/892979095 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec809a830 con 0x55ffa90c35f0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.459+0000 7f6ebb7fe700 1 -- 172.21.15.90:0/892979095 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ec8003e10 con 0x55ffa90c35f0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.459+0000 7f6ebb7fe700 1 -- 172.21.15.90:0/892979095 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec8099390 con 0x55ffa90c35f0 2024-04-06T14:19:23.460 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.459+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/892979095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 msgr2=0x55ffa90c59e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.461 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.459+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f6ec800cfe0 tx=0x7f6ec80099e0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/892979095 shutdown_connections 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c59e0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90b9370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/892979095 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b98b0 0x55ffa90c30b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/892979095 >> 172.21.15.90:0/892979095 conn(0x55ffa900f6a0 msgr2=0x55ffa90aba40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/892979095 shutdown_connections 2024-04-06T14:19:23.464 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/892979095 wait complete. 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 -- start start 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.460+0000 7f6ee39dcb80 1 Processor -- start 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c03c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90c0900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90c35f0 0x55ffa90c0e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa8f9dd80 con 0x55ffa90b8f90 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa8f9da20 con 0x55ffa90b98b0 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa8e856a0 con 0x55ffa90c35f0 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ee39dcb80 10 client.0 _ll_register_callbacks cb 0x55ffa8fa7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed4dc3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90c0900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c03c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed4dc3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90c0900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:36976/0 (socket says 172.21.15.90:36976) 2024-04-06T14:19:23.465 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed4dc3700 1 -- 172.21.15.90:0/27654384 learned_addr learned my addr 172.21.15.90:0/27654384 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90c35f0 0x55ffa90c0e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 -- 172.21.15.90:0/27654384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90c35f0 msgr2=0x55ffa90c0e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90c35f0 0x55ffa90c0e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 -- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 msgr2=0x55ffa90c0900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90c0900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 -- 172.21.15.90:0/27654384 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ed01a6410 con 0x55ffa90b8f90 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.461+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c03c0 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f6ecc01ebe0 tx=0x7f6ecc0047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f3baf2651e178c56 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6eb8ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ecc00ec50 con 0x55ffa90b8f90 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ffa8fc2380 con 0x55ffa90b8f90 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6ecc00edb0 con 0x55ffa90b8f90 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6ecc02f430 con 0x55ffa90b8f90 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6ee39dcb80 10 client.17724 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:23.466 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ffa8e2fb60 con 0x55ffa90b8f90 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.462+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ecc033020 con 0x55ffa90b8f90 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6ecc0263f0 con 0x55ffa90b8f90 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 10 client.17724 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 10 client.17724 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 20 client.17724 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa90d30d0 0x55ffa90d54f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 4 client.17724 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ffa8ffa700 con 0x55ffa90d30d0 2024-04-06T14:19:23.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.463+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa90d30d0 0x55ffa90d54f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.464+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa90d30d0 0x55ffa90d54f0 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7558b0704dee52a2 server_cookie=8204918eebd04f99 in_seq=0 out_seq=0 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.465+0000 7f6eb8ff9700 10 client.17724 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.465+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/27654384 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ffa8ffa700 con 0x55ffa90d30d0 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.465+0000 7f6eb8ff9700 10 client.17724 handle_command_reply: tid=0 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.465+0000 7f6ee39dcb80 1 client.17724 shutdown 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 msgr2=0x55ffa90c03c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c03c0 secure :-1 s=READY pgs=835 cs=0 l=1 rev1=1 crypto rx=0x7f6ecc01ebe0 tx=0x7f6ecc0047c0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 shutdown_connections 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa90d30d0 0x55ffa90d54f0 crc :-1 s=READY pgs=233 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90c35f0 0x55ffa90c0e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c03c0 unknown :-1 s=CLOSED pgs=835 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/27654384 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90c0900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 >> 172.21.15.90:0/27654384 conn(0x55ffa900f6a0 msgr2=0x55ffa90aac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6eb8ff9700 0 client.17724 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:23.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 shutdown_connections 2024-04-06T14:19:23.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/27654384 wait complete. 2024-04-06T14:19:23.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.466+0000 7f6ee39dcb80 20 client.17724 trim_cache size 0 max 16384 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.485+0000 7f6ee39dcb80 1 Processor -- start 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.485+0000 7f6ee39dcb80 1 -- start start 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90c0c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90ab3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 0x55ffa90ab8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa90b9f20 con 0x55ffa90c35f0 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa90be910 con 0x55ffa90b98b0 2024-04-06T14:19:23.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa90bbf80 con 0x55ffa90b8f90 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 0x55ffa90ab8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ed5dc5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 0x55ffa90ab8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54978/0 (socket says 172.21.15.90:54978) 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/2049555313 learned_addr learned my addr 172.21.15.90:0/2049555313 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90c0c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.486+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90ab3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 -- 172.21.15.90:0/2049555313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 msgr2=0x55ffa90c0c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90c0c60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 -- 172.21.15.90:0/2049555313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 msgr2=0x55ffa90ab8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 0x55ffa90ab8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 -- 172.21.15.90:0/2049555313 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ffa8fc7ea0 con 0x55ffa90b98b0 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90ab3a0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x55ffa90d34b0 tx=0x7f6ec0004320 comp rx=0 tx=0).ready entity=mon.2 client_cookie=43ef7729ad33fac7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ebbfff700 1 -- 172.21.15.90:0/2049555313 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec0016040 con 0x55ffa90b98b0 2024-04-06T14:19:23.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ebbfff700 1 -- 172.21.15.90:0/2049555313 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ec001bdc0 con 0x55ffa90b98b0 2024-04-06T14:19:23.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.487+0000 7f6ebbfff700 1 -- 172.21.15.90:0/2049555313 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec0003ea0 con 0x55ffa90b98b0 2024-04-06T14:19:23.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2049555313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 msgr2=0x55ffa90ab3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90ab3a0 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x55ffa90d34b0 tx=0x7f6ec0004320 comp rx=0 tx=0).stop 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2049555313 shutdown_connections 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90c35f0 0x55ffa90ab8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b8f90 0x55ffa90c0c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2049555313 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90b98b0 0x55ffa90ab3a0 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.488+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2049555313 >> 172.21.15.90:0/2049555313 conn(0x55ffa900f6a0 msgr2=0x55ffa90bec90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.489+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2049555313 shutdown_connections 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.489+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2049555313 wait complete. 2024-04-06T14:19:23.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.489+0000 7f6ee39dcb80 1 -- start start 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.489+0000 7f6ee39dcb80 1 Processor -- start 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c88d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b98b0 0x55ffa90c8e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c9350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9126f10 con 0x55ffa90b8f90 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9127050 con 0x55ffa90c35f0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ffa9127190 con 0x55ffa90b98b0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 10 client.0 _ll_register_callbacks cb 0x55ffa8fa7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed55c4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c88d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed55c4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c88d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:54986/0 (socket says 172.21.15.90:54986) 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed55c4700 1 -- 172.21.15.90:0/2843802285 learned_addr learned my addr 172.21.15.90:0/2843802285 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c9350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed4dc3700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b98b0 0x55ffa90c8e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/2843802285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b98b0 msgr2=0x55ffa90c8e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b98b0 0x55ffa90c8e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/2843802285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 msgr2=0x55ffa90c88d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c88d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 -- 172.21.15.90:0/2843802285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ec0007ca0 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ed5dc5700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c9350 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f6ec8009fd0 tx=0x7f6ec80092e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f6c95ce860c4aaa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6eb8ff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6ec809b330 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6ec801de40 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.490+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6ec8003e10 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6ec809ed00 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 10 client.27455 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ffa8e87cd0 con 0x55ffa90c35f0 2024-04-06T14:19:23.498 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6ec8003e10 con 0x55ffa90c35f0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6ec80b0d90 con 0x55ffa90c35f0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 10 client.27455 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 10 client.27455 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 20 client.27455 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa91285d0 0x55ffa912a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 4 client.27455 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ffa8e50cf0 con 0x55ffa91285d0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa91285d0 0x55ffa912a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.491+0000 7f6eb8ff9700 10 client.27455 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ed55c4700 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa91285d0 0x55ffa912a9f0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a380991f918a1682 server_cookie=79497c0729b77553 in_seq=0 out_seq=0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6eb8ff9700 1 -- 172.21.15.90:0/2843802285 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55ffa8e50cf0 con 0x55ffa91285d0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6eb8ff9700 10 client.27455 handle_command_reply: tid=0 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 client.27455 shutdown 2024-04-06T14:19:23.499 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 msgr2=0x55ffa90c9350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c9350 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f6ec8009fd0 tx=0x7f6ec80092e0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 shutdown_connections 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ffa90c35f0 0x55ffa90c9350 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ffa90b8f90 0x55ffa90c88d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ffa91285d0 0x55ffa912a9f0 crc :-1 s=READY pgs=234 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 --2- 172.21.15.90:0/2843802285 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ffa90b98b0 0x55ffa90c8e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6eb8ff9700 0 client.27455 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 >> 172.21.15.90:0/2843802285 conn(0x55ffa900f6a0 msgr2=0x55ffa90b7f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 shutdown_connections 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 1 -- 172.21.15.90:0/2843802285 wait complete. 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.492+0000 7f6ee39dcb80 20 client.27455 trim_cache size 0 max 16384 2024-04-06T14:19:23.500 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:23.501 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:19:23.501 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:19:23.501 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:23.501 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.493+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 msgr2=0x7f6ebc0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.493+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 0x7f6ebc0a3750 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f6ecc009c60 tx=0x7f6ecc009260 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.493+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 msgr2=0x7f6ed01998f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.493+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 secure :-1 s=READY pgs=834 cs=0 l=1 rev1=1 crypto rx=0x7f6ec800b070 tx=0x7f6ec8007cf0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 shutdown_connections 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f6ebc0a1290 0x7f6ebc0a3750 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6ed0199440 0x7f6ed01998f0 unknown :-1 s=CLOSED pgs=834 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6ed00691d0 0x7f6ed019f230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 --2- 172.21.15.90:0/1153865648 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6ed01060b0 0x7f6ed019f770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 >> 172.21.15.90:0/1153865648 conn(0x7f6ed00769b0 msgr2=0x7f6ed0104960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 shutdown_connections 2024-04-06T14:19:23.502 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.495+0000 7f6ed7828700 1 -- 172.21.15.90:0/1153865648 wait complete. 2024-04-06T14:19:23.503 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:23 smithi105 ceph-mon[46969]: pgmap v438: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 10 MiB/s wr, 614 op/s 2024-04-06T14:19:23.509 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:23 smithi090 ceph-mon[43420]: pgmap v438: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 10 MiB/s wr, 614 op/s 2024-04-06T14:19:23.510 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:19:23.510 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:19:23.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:23 smithi134 ceph-mon[47549]: pgmap v438: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 10 MiB/s wr, 614 op/s 2024-04-06T14:19:23.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.751+0000 7f93cb651700 1 -- 172.21.15.90:0/18604927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410d0c0 msgr2=0x7f93c410d540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.754 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.751+0000 7f93cb651700 1 --2- 172.21.15.90:0/18604927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410d0c0 0x7f93c410d540 secure :-1 s=READY pgs=836 cs=0 l=1 rev1=1 crypto rx=0x7f93c0009990 tx=0x7f93c0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.755+0000 7f93cb651700 1 -- 172.21.15.90:0/18604927 shutdown_connections 2024-04-06T14:19:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.755+0000 7f93cb651700 1 --2- 172.21.15.90:0/18604927 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c4107db0 0x7f93c4108230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.755+0000 7f93cb651700 1 --2- 172.21.15.90:0/18604927 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c410f3d0 0x7f93c410f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.755+0000 7f93cb651700 1 --2- 172.21.15.90:0/18604927 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410d0c0 0x7f93c410d540 unknown :-1 s=CLOSED pgs=836 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.759 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.755+0000 7f93cb651700 1 -- 172.21.15.90:0/18604927 >> 172.21.15.90:0/18604927 conn(0x7f93c406bd30 msgr2=0x7f93c406c140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:23.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.760+0000 7f93cb651700 1 -- 172.21.15.90:0/18604927 shutdown_connections 2024-04-06T14:19:23.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.762+0000 7f93cb651700 1 -- 172.21.15.90:0/18604927 wait complete. 2024-04-06T14:19:23.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.762+0000 7f93cb651700 1 Processor -- start 2024-04-06T14:19:23.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 -- start start 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c410d0c0 0x7f93c411dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410f3d0 0x7f93c4116c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93c4117440 con 0x7f93c410f3d0 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93c41175b0 con 0x7f93c410d0c0 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93cb651700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93c4117770 con 0x7f93c4107db0 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93c93ed700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93c93ed700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48604/0 (socket says 172.21.15.90:48604) 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93c93ed700 1 -- 172.21.15.90:0/2848890764 learned_addr learned my addr 172.21.15.90:0/2848890764 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:23.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.764+0000 7f93c8bec700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c410d0c0 0x7f93c411dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 -- 172.21.15.90:0/2848890764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c410d0c0 msgr2=0x7f93c411dfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c410d0c0 0x7f93c411dfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 -- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410f3d0 msgr2=0x7f93c4116c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410f3d0 0x7f93c4116c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:23.766 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 -- 172.21.15.90:0/2848890764 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93c00095f0 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93c93ed700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f93b400b550 tx=0x7f93b400b860 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93b4011830 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f93c411b580 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f93b4011e50 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f93c411ba60 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.765+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f93b400f2e0 con 0x7f93c4107db0 2024-04-06T14:19:23.767 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.767+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f93a80052f0 con 0x7f93c4107db0 2024-04-06T14:19:23.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.768+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f93b4011990 con 0x7f93c4107db0 2024-04-06T14:19:23.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.768+0000 7f93ba7fc700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 0x7f93b00a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:23.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.768+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f93b40cc5c0 con 0x7f93c4107db0 2024-04-06T14:19:23.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.768+0000 7f93c8bec700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 0x7f93b00a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:23.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.769+0000 7f93c8bec700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 0x7f93b00a3890 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f93c4118330 tx=0x7f93c000b540 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:23.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:23.773+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f93b4095aa0 con 0x7f93c4107db0 2024-04-06T14:19:24.130 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.127+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f93a8005160 con 0x7f93c4107db0 2024-04-06T14:19:24.137 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:19:24.137 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:19:24.137 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.134+0000 7f93ba7fc700 1 -- 172.21.15.90:0/2848890764 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f93b40990c0 con 0x7f93c4107db0 2024-04-06T14:19:24.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.142+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 msgr2=0x7f93b00a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.142+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 0x7f93b00a3890 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f93c4118330 tx=0x7f93c000b540 comp rx=0 tx=0).stop 2024-04-06T14:19:24.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.142+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 msgr2=0x7f93c411da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.142+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f93b400b550 tx=0x7f93b400b860 comp rx=0 tx=0).stop 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 shutdown_connections 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f93b00a13d0 0x7f93b00a3890 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f93c410f3d0 0x7f93c4116c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f93c4107db0 0x7f93c411da60 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 --2- 172.21.15.90:0/2848890764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f93c410d0c0 0x7f93c411dfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 >> 172.21.15.90:0/2848890764 conn(0x7f93c406bd30 msgr2=0x7f93c410acf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 shutdown_connections 2024-04-06T14:19:24.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.143+0000 7f93cb651700 1 -- 172.21.15.90:0/2848890764 wait complete. 2024-04-06T14:19:24.147 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:19:24.163 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:24.163 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:19:24.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.372+0000 7f647fb07700 1 -- 172.21.15.90:0/2402294060 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647810f920 msgr2=0x7f64781082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.372+0000 7f647fb07700 1 --2- 172.21.15.90:0/2402294060 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647810f920 0x7f64781082f0 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f647000b0a0 tx=0x7f647000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.375+0000 7f647fb07700 1 -- 172.21.15.90:0/2402294060 shutdown_connections 2024-04-06T14:19:24.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.375+0000 7f647fb07700 1 --2- 172.21.15.90:0/2402294060 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647811a590 0x7f647811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.375+0000 7f647fb07700 1 --2- 172.21.15.90:0/2402294060 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f4b0 0x7f6478107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.375+0000 7f647fb07700 1 --2- 172.21.15.90:0/2402294060 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647810f920 0x7f64781082f0 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.375+0000 7f647fb07700 1 -- 172.21.15.90:0/2402294060 >> 172.21.15.90:0/2402294060 conn(0x7f647806be00 msgr2=0x7f647806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.380+0000 7f647fb07700 1 -- 172.21.15.90:0/2402294060 shutdown_connections 2024-04-06T14:19:24.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.380+0000 7f647fb07700 1 -- 172.21.15.90:0/2402294060 wait complete. 2024-04-06T14:19:24.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 Processor -- start 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 -- start start 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f920 0x7f647811a050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647811a590 0x7f6478113e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f64781145e0 con 0x7f647810f920 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6478114750 con 0x7f647811a590 2024-04-06T14:19:24.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.381+0000 7f647fb07700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6478114910 con 0x7f647810f4b0 2024-04-06T14:19:24.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48628/0 (socket says 172.21.15.90:48628) 2024-04-06T14:19:24.383 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 -- 172.21.15.90:0/920014380 learned_addr learned my addr 172.21.15.90:0/920014380 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d0a2700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f920 0x7f647811a050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 -- 172.21.15.90:0/920014380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647811a590 msgr2=0x7f6478113e10 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647811a590 0x7f6478113e10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 -- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f920 msgr2=0x7f647811a050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f920 0x7f647811a050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.382+0000 7f647d8a3700 1 -- 172.21.15.90:0/920014380 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6470009d00 con 0x7f647810f4b0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f647d8a3700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f647400b530 tx=0x7f647400b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f646affd700 1 -- 172.21.15.90:0/920014380 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6474011830 con 0x7f647810f4b0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f646affd700 1 -- 172.21.15.90:0/920014380 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6474011e50 con 0x7f647810f4b0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f64781bb200 con 0x7f647810f4b0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64781bb680 con 0x7f647810f4b0 2024-04-06T14:19:24.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.383+0000 7f646affd700 1 -- 172.21.15.90:0/920014380 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f647400f2e0 con 0x7f647810f4b0 2024-04-06T14:19:24.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.386+0000 7f646affd700 1 -- 172.21.15.90:0/920014380 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f647400f440 con 0x7f647810f4b0 2024-04-06T14:19:24.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.387+0000 7f646affd700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 0x7f64640a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.387+0000 7f646affd700 1 -- 172.21.15.90:0/920014380 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f64740ccb00 con 0x7f647810f4b0 2024-04-06T14:19:24.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.387+0000 7f648bcbbb80 1 Processor -- start 2024-04-06T14:19:24.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 -- start start 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd00ade0 0x55e1fd00d1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fcf58990 con 0x55e1fd000780 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fcf56350 con 0x55e1fd00ade0 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fcf51c40 con 0x55e1fd0010a0 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d0a2700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 0x7f64640a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55018/0 (socket says 172.21.15.90:55018) 2024-04-06T14:19:24.389 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 -- 172.21.15.90:0/1441990220 learned_addr learned my addr 172.21.15.90:0/1441990220 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.390 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647e0a4700 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd00ade0 0x55e1fd00d1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d0a2700 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 -- 172.21.15.90:0/1441990220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 msgr2=0x55e1fd00a8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00a8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 -- 172.21.15.90:0/1441990220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd00ade0 msgr2=0x55e1fd00d1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd00ade0 0x55e1fd00d1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.388+0000 7f647d8a3700 1 -- 172.21.15.90:0/1441990220 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e1fcdd33b0 con 0x55e1fd000780 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.389+0000 7f647d8a3700 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f64740a1bf0 tx=0x7f64740996d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3c0c8d2ac7de9053 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.389+0000 7f647d0a2700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 0x7f64640a3810 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f6470009fd0 tx=0x7f6470009400 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.389+0000 7f6463fff700 1 -- 172.21.15.90:0/1441990220 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f647409dad0 con 0x55e1fd000780 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.389+0000 7f6463fff700 1 -- 172.21.15.90:0/1441990220 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f64740a2120 con 0x55e1fd000780 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.389+0000 7f6463fff700 1 -- 172.21.15.90:0/1441990220 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64740993f0 con 0x55e1fd000780 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1441990220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 msgr2=0x55e1fd000b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 secure :-1 s=READY pgs=837 cs=0 l=1 rev1=1 crypto rx=0x7f64740a1bf0 tx=0x7f64740996d0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1441990220 shutdown_connections 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd00ade0 0x55e1fd00d1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd000b60 unknown :-1 s=CLOSED pgs=837 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1441990220 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00a8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1441990220 >> 172.21.15.90:0/1441990220 conn(0x55e1fcf5f6a0 msgr2=0x55e1fcffba80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1441990220 shutdown_connections 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.390+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1441990220 wait complete. 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 -- start start 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 Processor -- start 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd0010a0 0x55e1fd007b40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd008170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fceedd80 con 0x55e1fd000780 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fceeda20 con 0x55e1fd0010a0 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fcdd56a0 con 0x55e1fd00ade0 2024-04-06T14:19:24.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f648bcbbb80 10 client.0 _ll_register_callbacks cb 0x55e1fcef7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.394 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55024/0 (socket says 172.21.15.90:55024) 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 -- 172.21.15.90:0/2639544372 learned_addr learned my addr 172.21.15.90:0/2639544372 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd0010a0 0x55e1fd007b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647e0a4700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd008170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 -- 172.21.15.90:0/2639544372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 msgr2=0x55e1fd008170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd008170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 -- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd0010a0 msgr2=0x55e1fd007b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd0010a0 0x55e1fd007b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.391+0000 7f647d8a3700 1 -- 172.21.15.90:0/2639544372 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f64781bb680 con 0x55e1fd000780 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.392+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7f647400b500 tx=0x7f6474009380 comp rx=0 tx=0).ready entity=mon.0 client_cookie=633b87b425a8c91 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.392+0000 7f64617fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:24.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.392+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64740993a0 con 0x55e1fd000780 2024-04-06T14:19:24.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.392+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6474099f00 con 0x55e1fd000780 2024-04-06T14:19:24.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.394+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e1fcf12380 con 0x55e1fd000780 2024-04-06T14:19:24.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.397+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f647409a0c0 con 0x55e1fd000780 2024-04-06T14:19:24.397 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.397+0000 7f648bcbbb80 10 client.17742 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.398 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.397+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55e1fcd7fb60 con 0x55e1fd000780 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.399+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e1fcf12380 con 0x55e1fd000780 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.399+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f64740adb70 con 0x55e1fd000780 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.399+0000 7f648bcbbb80 10 client.17742 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.399+0000 7f648bcbbb80 10 client.17742 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.400+0000 7f648bcbbb80 20 client.17742 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.400+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd01a560 0x55e1fd01c950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.400 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.400+0000 7f648bcbbb80 4 client.17742 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:24.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.400+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e1fcf4a700 con 0x55e1fd01a560 2024-04-06T14:19:24.401 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.400+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd01a560 0x55e1fd01c950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.403+0000 7f64617fa700 10 client.17742 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:24.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.403+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd01a560 0x55e1fd01c950 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=13fd18fd4af347ef server_cookie=c4f194ce0cfdc80d in_seq=0 out_seq=0 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f64617fa700 1 -- 172.21.15.90:0/2639544372 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55e1fcf4a700 con 0x55e1fd01a560 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f64617fa700 10 client.17742 handle_command_reply: tid=0 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 client.17742 shutdown 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 msgr2=0x55e1fd00a800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 secure :-1 s=READY pgs=838 cs=0 l=1 rev1=1 crypto rx=0x7f647400b500 tx=0x7f6474009380 comp rx=0 tx=0).stop 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 shutdown_connections 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd01a560 0x55e1fd01c950 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd008170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.405 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f64617fa700 0 client.17742 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd000780 0x55e1fd00a800 unknown :-1 s=CLOSED pgs=838 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2639544372 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd0010a0 0x55e1fd007b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 >> 172.21.15.90:0/2639544372 conn(0x55e1fcf5f6a0 msgr2=0x55e1fcffac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 shutdown_connections 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.404+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2639544372 wait complete. 2024-04-06T14:19:24.406 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.405+0000 7f648bcbbb80 20 client.17742 trim_cache size 0 max 16384 2024-04-06T14:19:24.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 Processor -- start 2024-04-06T14:19:24.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 -- start start 2024-04-06T14:19:24.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd007b00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.423 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd00a330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd006090 con 0x55e1fd0010a0 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd003730 con 0x55e1fd000780 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd008750 con 0x55e1fd00ade0 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647e0a4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd00a330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55038/0 (socket says 172.21.15.90:55038) 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 -- 172.21.15.90:0/1974906154 learned_addr learned my addr 172.21.15.90:0/1974906154 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d8a3700 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd007b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.424 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 -- 172.21.15.90:0/1974906154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 msgr2=0x55e1fd00a330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd00a330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 -- 172.21.15.90:0/1974906154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 msgr2=0x55e1fd007b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd007b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647d0a2700 1 -- 172.21.15.90:0/1974906154 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e1fcf17ea0 con 0x55e1fd0010a0 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.422+0000 7f647e0a4700 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd00a330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f647d0a2700 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f647000d010 tx=0x7f6470022000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=875841eac94063a8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f6468da5700 1 -- 172.21.15.90:0/1974906154 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6470030cd0 con 0x55e1fd0010a0 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f6468da5700 1 -- 172.21.15.90:0/1974906154 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6470030e30 con 0x55e1fd0010a0 2024-04-06T14:19:24.425 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f6468da5700 1 -- 172.21.15.90:0/1974906154 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f64700206b0 con 0x55e1fd0010a0 2024-04-06T14:19:24.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1974906154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 msgr2=0x55e1fd011ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 secure :-1 s=READY pgs=839 cs=0 l=1 rev1=1 crypto rx=0x7f647000d010 tx=0x7f6470022000 comp rx=0 tx=0).stop 2024-04-06T14:19:24.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1974906154 shutdown_connections 2024-04-06T14:19:24.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd00ade0 0x55e1fd00a330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.428 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd007b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.423+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/1974906154 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd0010a0 0x55e1fd011ac0 unknown :-1 s=CLOSED pgs=839 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1974906154 >> 172.21.15.90:0/1974906154 conn(0x55e1fcf5f6a0 msgr2=0x55e1fd0212d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1974906154 shutdown_connections 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- 172.21.15.90:0/1974906154 wait complete. 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- start start 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 Processor -- start 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00ff00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd00ade0 0x55e1fd010460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd010bc0 con 0x55e1fd00ade0 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd010d30 con 0x55e1fd000780 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e1fd06e390 con 0x55e1fd0010a0 2024-04-06T14:19:24.429 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.424+0000 7f648bcbbb80 10 client.0 _ll_register_callbacks cb 0x55e1fcef7470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647e0a4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd00ade0 0x55e1fd010460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37072/0 (socket says 172.21.15.90:37072) 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 -- 172.21.15.90:0/2763625197 learned_addr learned my addr 172.21.15.90:0/2763625197 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00ff00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 -- 172.21.15.90:0/2763625197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 msgr2=0x55e1fd00ff00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00ff00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 -- 172.21.15.90:0/2763625197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd00ade0 msgr2=0x55e1fd010460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd00ade0 0x55e1fd010460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 -- 172.21.15.90:0/2763625197 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f646c009560 con 0x55e1fd000780 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f647d8a3700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f64740961a0 tx=0x7f64740996d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=24bcc23ab5f4bd64 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f64617fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6474018310 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f647409a3a0 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.425+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f64740a2120 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.426+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6474095a90 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.426+0000 7f648bcbbb80 10 client.27473 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.426+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f647811d270 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.426+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f64740a2120 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6474099f00 con 0x55e1fd000780 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 10 client.27473 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 10 client.27473 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 20 client.27473 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd06f600 0x55e1fd071a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 4 client.27473 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:24.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.427+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55e1fcda0cf0 con 0x55e1fd06f600 2024-04-06T14:19:24.432 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.428+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd06f600 0x55e1fd071a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.428+0000 7f647d0a2700 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd06f600 0x55e1fd071a20 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=2daa42e5d373baa0 server_cookie=5869b5cca697d58c in_seq=0 out_seq=0 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.428+0000 7f64617fa700 10 client.27473 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f64617fa700 1 -- 172.21.15.90:0/2763625197 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55e1fcda0cf0 con 0x55e1fd06f600 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f64617fa700 10 client.27473 handle_command_reply: tid=0 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 client.27473 shutdown 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 msgr2=0x55e1fd011a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7f64740961a0 tx=0x7f64740996d0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 shutdown_connections 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55e1fd06f600 0x55e1fd071a20 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.433 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55e1fd00ade0 0x55e1fd010460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55e1fd000780 0x55e1fd011a10 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 --2- 172.21.15.90:0/2763625197 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55e1fd0010a0 0x55e1fd00ff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 >> 172.21.15.90:0/2763625197 conn(0x55e1fcf5f6a0 msgr2=0x55e1fcffb500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f64617fa700 0 client.27473 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 shutdown_connections 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 1 -- 172.21.15.90:0/2763625197 wait complete. 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.429+0000 7f648bcbbb80 20 client.27473 trim_cache size 0 max 16384 2024-04-06T14:19:24.434 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 msgr2=0x7f64640a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 0x7f64640a3810 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f6470009fd0 tx=0x7f6470009400 comp rx=0 tx=0).stop 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 msgr2=0x7f6478119b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f647400b530 tx=0x7f647400b8f0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 shutdown_connections 2024-04-06T14:19:24.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f64640a1350 0x7f64640a3810 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f647811a590 0x7f6478113e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f647810f4b0 0x7f6478119b10 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 --2- 172.21.15.90:0/920014380 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f647810f920 0x7f647811a050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 >> 172.21.15.90:0/920014380 conn(0x7f647806be00 msgr2=0x7f647811a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 shutdown_connections 2024-04-06T14:19:24.436 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.430+0000 7f647fb07700 1 -- 172.21.15.90:0/920014380 wait complete. 2024-04-06T14:19:24.444 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:24 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2848890764' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:24.445 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:24.446 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:19:24.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:24 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2848890764' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 -- 172.21.15.90:0/2955512462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ff1d0 msgr2=0x7f60140ff650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 --2- 172.21.15.90:0/2955512462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ff1d0 0x7f60140ff650 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f6010009990 tx=0x7f6010009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 -- 172.21.15.90:0/2955512462 shutdown_connections 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 --2- 172.21.15.90:0/2955512462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ffb90 0x7f601410be40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 --2- 172.21.15.90:0/2955512462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f6014104320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 --2- 172.21.15.90:0/2955512462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ff1d0 0x7f60140ff650 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 -- 172.21.15.90:0/2955512462 >> 172.21.15.90:0/2955512462 conn(0x7f6014074b50 msgr2=0x7f6014074f60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 -- 172.21.15.90:0/2955512462 shutdown_connections 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 -- 172.21.15.90:0/2955512462 wait complete. 2024-04-06T14:19:24.636 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.633+0000 7f601bcab700 1 Processor -- start 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 -- start start 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ff1d0 0x7f6014108e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 0x7f60141093a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f60141b1ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60141b20a0 con 0x7f60140ff1d0 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f601406a800 con 0x7f6014103f40 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601bcab700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f601406a9c0 con 0x7f60140ffb90 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f6019246700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 0x7f60141093a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f6019a47700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ff1d0 0x7f6014108e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f60141b1ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f6019246700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 0x7f60141093a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48688/0 (socket says 172.21.15.90:48688) 2024-04-06T14:19:24.637 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f6019246700 1 -- 172.21.15.90:0/1972843951 learned_addr learned my addr 172.21.15.90:0/1972843951 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 -- 172.21.15.90:0/1972843951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 msgr2=0x7f60141093a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 0x7f60141093a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 -- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ff1d0 msgr2=0x7f6014108e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ff1d0 0x7f6014108e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 -- 172.21.15.90:0/1972843951 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60100095f0 con 0x7f6014103f40 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f601a248700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f60141b1ad0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f600c007f10 tx=0x7f600c00d3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.634+0000 7f600affd700 1 -- 172.21.15.90:0/1972843951 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f600c004db0 con 0x7f6014103f40 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.635+0000 7f600affd700 1 -- 172.21.15.90:0/1972843951 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f600c00f040 con 0x7f6014103f40 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.635+0000 7f600affd700 1 -- 172.21.15.90:0/1972843951 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f600c005990 con 0x7f6014103f40 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.635+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f601406aca0 con 0x7f6014103f40 2024-04-06T14:19:24.638 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.635+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f601406b160 con 0x7f6014103f40 2024-04-06T14:19:24.639 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.639+0000 7f6027e5fb80 1 Processor -- start 2024-04-06T14:19:24.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.639+0000 7f600affd700 1 -- 172.21.15.90:0/1972843951 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f600c00db20 con 0x7f6014103f40 2024-04-06T14:19:24.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 -- start start 2024-04-06T14:19:24.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b13b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b201d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506a6b990 con 0x557506b140a0 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506a69350 con 0x557506b1dde0 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506a64c40 con 0x557506b13780 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6019a47700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f600affd700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 0x7f60000a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6019a47700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55090/0 (socket says 172.21.15.90:55090) 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6019a47700 1 -- 172.21.15.90:0/3564670778 learned_addr learned my addr 172.21.15.90:0/3564670778 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f600affd700 1 -- 172.21.15.90:0/1972843951 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f600c0cc5f0 con 0x7f6014103f40 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6019246700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b201d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.640+0000 7f6019246700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b13b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.642 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 -- 172.21.15.90:0/3564670778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 msgr2=0x557506b13b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b13b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 -- 172.21.15.90:0/3564670778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 msgr2=0x557506b201d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b201d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 -- 172.21.15.90:0/3564670778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5575068e63b0 con 0x557506b140a0 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019246700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b13b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:19:24.643 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f6019a47700 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f6004009d80 tx=0x7f6004011720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=345038282bed6c85 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:24 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2848890764' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f5ffffff700 1 -- 172.21.15.90:0/3564670778 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004004230 con 0x557506b140a0 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f5ffffff700 1 -- 172.21.15.90:0/3564670778 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6004004390 con 0x557506b140a0 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.641+0000 7f5ffffff700 1 -- 172.21.15.90:0/3564670778 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004019880 con 0x557506b140a0 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f601a248700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 0x7f60000a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 -- 172.21.15.90:0/3564670778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 msgr2=0x557506b1d8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 secure :-1 s=READY pgs=840 cs=0 l=1 rev1=1 crypto rx=0x7f6004009d80 tx=0x7f6004011720 comp rx=0 tx=0).stop 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 -- 172.21.15.90:0/3564670778 shutdown_connections 2024-04-06T14:19:24.647 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b201d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b13b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/3564670778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b140a0 0x557506b1d8a0 unknown :-1 s=CLOSED pgs=840 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 -- 172.21.15.90:0/3564670778 >> 172.21.15.90:0/3564670778 conn(0x557506a726a0 msgr2=0x557506b0ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 -- 172.21.15.90:0/3564670778 shutdown_connections 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f601a248700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 0x7f60000a3750 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f600c00d700 tx=0x7f600c0053b0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.642+0000 7f6027e5fb80 1 -- 172.21.15.90:0/3564670778 wait complete. 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.643+0000 7f6027e5fb80 1 -- start start 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.643+0000 7f6027e5fb80 1 Processor -- start 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1ac00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b1b140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b1b680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506a00d80 con 0x557506b13780 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506a00a20 con 0x557506b1dde0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5575068e86a0 con 0x557506b140a0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6027e5fb80 10 client.0 _ll_register_callbacks cb 0x557506a0a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1ac00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1ac00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55100/0 (socket says 172.21.15.90:55100) 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019246700 1 -- 172.21.15.90:0/2997521953 learned_addr learned my addr 172.21.15.90:0/2997521953 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b1b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019246700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b1b680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 -- 172.21.15.90:0/2997521953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 msgr2=0x557506b1b680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b1b680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 -- 172.21.15.90:0/2997521953 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 msgr2=0x557506b1ac00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1ac00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.644+0000 7f6019a47700 1 -- 172.21.15.90:0/2997521953 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60040099e0 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6019a47700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b1b140 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f6004009d80 tx=0x7f60040038b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b72b25100672afb3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004005450 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6004005ce0 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557506a25380 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6004009ed0 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 10 client.27223 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x557506892b60 con 0x557506b140a0 2024-04-06T14:19:24.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557506a25380 con 0x557506b140a0 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6004021040 con 0x557506b140a0 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 10 client.27223 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 10 client.27223 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 20 client.27223 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b2d850 0x557506b2fc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 4 client.27223 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.645+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557506a5d700 con 0x557506b2d850 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.646+0000 7f6019246700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b2d850 0x557506b2fc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.646+0000 7f5ffd7fa700 10 client.27223 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:24.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.646+0000 7f6019246700 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b2d850 0x557506b2fc70 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=cd9773a9edfad084 server_cookie=ef0c3f3b5de9810a in_seq=0 out_seq=0 2024-04-06T14:19:24.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.647+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2997521953 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x557506a5d700 con 0x557506b2d850 2024-04-06T14:19:24.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.647+0000 7f5ffd7fa700 10 client.27223 handle_command_reply: tid=0 2024-04-06T14:19:24.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.647+0000 7f6027e5fb80 1 client.27223 shutdown 2024-04-06T14:19:24.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.647+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 msgr2=0x557506b1b140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.647+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b1b140 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f6004009d80 tx=0x7f60040038b0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 shutdown_connections 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b2d850 0x557506b2fc70 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b1b680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1ac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2997521953 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b1b140 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 >> 172.21.15.90:0/2997521953 conn(0x557506a726a0 msgr2=0x557506b0dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f5ffd7fa700 0 client.27223 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 shutdown_connections 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2997521953 wait complete. 2024-04-06T14:19:24.653 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.652+0000 7f6027e5fb80 20 client.27223 trim_cache size 0 max 16384 2024-04-06T14:19:24.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 Processor -- start 2024-04-06T14:19:24.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 -- start start 2024-04-06T14:19:24.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b19be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b0e3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b1dbf0 con 0x557506b13780 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b19150 con 0x557506b1dde0 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b167d0 con 0x557506b140a0 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55104/0 (socket says 172.21.15.90:55104) 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 -- 172.21.15.90:0/2690302000 learned_addr learned my addr 172.21.15.90:0/2690302000 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019a47700 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b19be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b0e3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 -- 172.21.15.90:0/2690302000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 msgr2=0x557506b19be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b19be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 -- 172.21.15.90:0/2690302000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 msgr2=0x557506b0e3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b0e3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.670+0000 7f6019246700 1 -- 172.21.15.90:0/2690302000 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557506a2aea0 con 0x557506b13780 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6019246700 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7f6010006010 tx=0x7f601001a040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a07291590181407f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6008da5700 1 -- 172.21.15.90:0/2690302000 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f601002eb40 con 0x557506b13780 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6008da5700 1 -- 172.21.15.90:0/2690302000 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f601002e580 con 0x557506b13780 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6008da5700 1 -- 172.21.15.90:0/2690302000 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f601002aae0 con 0x557506b13780 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2690302000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 msgr2=0x557506b1b4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 secure :-1 s=READY pgs=841 cs=0 l=1 rev1=1 crypto rx=0x7f6010006010 tx=0x7f601001a040 comp rx=0 tx=0).stop 2024-04-06T14:19:24.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2690302000 shutdown_connections 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.671+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b1dde0 0x557506b0e3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b13780 0x557506b1b4a0 unknown :-1 s=CLOSED pgs=841 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2690302000 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b140a0 0x557506b19be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2690302000 >> 172.21.15.90:0/2690302000 conn(0x557506a726a0 msgr2=0x557506b194d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2690302000 shutdown_connections 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2690302000 wait complete. 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 -- start start 2024-04-06T14:19:24.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.672+0000 7f6027e5fb80 1 Processor -- start 2024-04-06T14:19:24.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b22800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b1dde0 0x557506b80680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.678 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b23490 con 0x557506b1dde0 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b80bc0 con 0x557506b140a0 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557506b80d80 con 0x557506b13780 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6027e5fb80 10 client.0 _ll_register_callbacks cb 0x557506a0a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37126/0 (socket says 172.21.15.90:37126) 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 -- 172.21.15.90:0/2905384580 learned_addr learned my addr 172.21.15.90:0/2905384580 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 -- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 msgr2=0x557506b22800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b22800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 -- 172.21.15.90:0/2905384580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b1dde0 msgr2=0x557506b80680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b1dde0 0x557506b80680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.673+0000 7f6019a47700 1 -- 172.21.15.90:0/2905384580 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6010020040 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f6019a47700 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x557506b0f8e0 tx=0x7f6004019420 comp rx=0 tx=0).ready entity=mon.2 client_cookie=82825cac6f1d1f21 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6004005450 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6004004390 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f6004003da0 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6004011b20 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f6027e5fb80 10 client.27491 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5575068eacd0 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6004004390 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.674+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f6004021070 con 0x557506b140a0 2024-04-06T14:19:24.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 10 client.27491 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 10 client.27491 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 20 client.27491 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b820d0 0x557506b844f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 4 client.27491 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5575068b3cf0 con 0x557506b820d0 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6019246700 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b820d0 0x557506b844f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f5ffd7fa700 10 client.27491 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6019246700 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b820d0 0x557506b844f0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=fd7edd49b1ecab8d server_cookie=a7f87fd637422f58 in_seq=0 out_seq=0 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f5ffd7fa700 1 -- 172.21.15.90:0/2905384580 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5575068b3cf0 con 0x557506b820d0 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f5ffd7fa700 10 client.27491 handle_command_reply: tid=0 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.675+0000 7f6027e5fb80 1 client.27491 shutdown 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 msgr2=0x557506b22d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x557506b0f8e0 tx=0x7f6004019420 comp rx=0 tx=0).stop 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 shutdown_connections 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x557506b820d0 0x557506b844f0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557506b1dde0 0x557506b80680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f5ffd7fa700 0 client.27491 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557506b13780 0x557506b22800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 --2- 172.21.15.90:0/2905384580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557506b140a0 0x557506b22d40 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 >> 172.21.15.90:0/2905384580 conn(0x557506a726a0 msgr2=0x557506b12770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 shutdown_connections 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 1 -- 172.21.15.90:0/2905384580 wait complete. 2024-04-06T14:19:24.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f6027e5fb80 20 client.27491 trim_cache size 0 max 16384 2024-04-06T14:19:24.683 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 msgr2=0x7f60000a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 0x7f60000a3750 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f600c00d700 tx=0x7f600c0053b0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 msgr2=0x7f60141b1ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f60141b1ad0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7f600c007f10 tx=0x7f600c00d3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 shutdown_connections 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f60000a1290 0x7f60000a3750 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6014103f40 0x7f60141b1ad0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f60140ff1d0 0x7f6014108e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.676+0000 7f601bcab700 1 --2- 172.21.15.90:0/1972843951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f60140ffb90 0x7f60141093a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.677+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 >> 172.21.15.90:0/1972843951 conn(0x7f6014074b50 msgr2=0x7f601410a750 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.677+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 shutdown_connections 2024-04-06T14:19:24.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.677+0000 7f601bcab700 1 -- 172.21.15.90:0/1972843951 wait complete. 2024-04-06T14:19:24.692 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:24.692 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:19:24.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 -- 172.21.15.90:0/4154492685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069bc0 msgr2=0x7fb214105980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/4154492685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069bc0 0x7fb214105980 secure :-1 s=READY pgs=842 cs=0 l=1 rev1=1 crypto rx=0x7fb204009990 tx=0x7fb204009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 -- 172.21.15.90:0/4154492685 shutdown_connections 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/4154492685 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb21410dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/4154492685 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb214069210 0x7fb2140695f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/4154492685 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069bc0 0x7fb214105980 unknown :-1 s=CLOSED pgs=842 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.877+0000 7fb21ba4c700 1 -- 172.21.15.90:0/4154492685 >> 172.21.15.90:0/4154492685 conn(0x7fb2140769b0 msgr2=0x7fb214076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- 172.21.15.90:0/4154492685 shutdown_connections 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- 172.21.15.90:0/4154492685 wait complete. 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 Processor -- start 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- start start 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 0x7fb214072550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb214072a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb21406c750 0x7fb21406cc00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.881 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb21406d3d0 con 0x7fb214069210 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2141ac2e0 con 0x7fb21406c750 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.878+0000 7fb21ba4c700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2141ac420 con 0x7fb2141060b0 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb2197e8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 0x7fb214072550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb2197e8700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 0x7fb214072550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55134/0 (socket says 172.21.15.90:55134) 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb2197e8700 1 -- 172.21.15.90:0/1014157431 learned_addr learned my addr 172.21.15.90:0/1014157431 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb214072a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb219fe9700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb21406c750 0x7fb21406cc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 -- 172.21.15.90:0/1014157431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb21406c750 msgr2=0x7fb21406cc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb21406c750 0x7fb21406cc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 -- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 msgr2=0x7fb214072550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 0x7fb214072550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 -- 172.21.15.90:0/1014157431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2040095f0 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb214072a90 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fb204000c00 tx=0x7fb204004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb20a7fc700 1 -- 172.21.15.90:0/1014157431 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20401d070 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb20a7fc700 1 -- 172.21.15.90:0/1014157431 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb204021410 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2141ac670 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb20a7fc700 1 -- 172.21.15.90:0/1014157431 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb204017650 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.879+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2141aca90 con 0x7fb2141060b0 2024-04-06T14:19:24.883 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.881+0000 7fb20a7fc700 1 -- 172.21.15.90:0/1014157431 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb204021580 con 0x7fb2141060b0 2024-04-06T14:19:24.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.885+0000 7fb20a7fc700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 0x7fb2000a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.885+0000 7fb20a7fc700 1 -- 172.21.15.90:0/1014157431 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb2040cdeb0 con 0x7fb2141060b0 2024-04-06T14:19:24.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 Processor -- start 2024-04-06T14:19:24.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 -- start start 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb2197e8700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 0x7fb2000a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4153a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc406c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc35e990 con 0x55d3cc414fc0 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc35c350 con 0x55d3cc407140 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb227c00b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc357c40 con 0x55d3cc406780 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37142/0 (socket says 172.21.15.90:37142) 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb219fe9700 1 -- 172.21.15.90:0/2777084924 learned_addr learned my addr 172.21.15.90:0/2777084924 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.886+0000 7fb218fe7700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc406c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.888 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb2197e8700 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4153a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 -- 172.21.15.90:0/2777084924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 msgr2=0x55d3cc406c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc406c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 -- 172.21.15.90:0/2777084924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 msgr2=0x55d3cc4153a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4153a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 -- 172.21.15.90:0/2777084924 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d3cc1d93b0 con 0x55d3cc407140 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb218fe7700 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc406c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb2197e8700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 0x7fb2000a3750 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fb210009e40 tx=0x7fb210009980 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fb20c011680 tx=0x7fb20c009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=65cb1d38dcac0c7d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb1ff7fe700 1 -- 172.21.15.90:0/2777084924 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20c00e040 con 0x55d3cc407140 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb1ff7fe700 1 -- 172.21.15.90:0/2777084924 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb20c00f040 con 0x55d3cc407140 2024-04-06T14:19:24.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.887+0000 7fb1ff7fe700 1 -- 172.21.15.90:0/2777084924 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20c004860 con 0x55d3cc407140 2024-04-06T14:19:24.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 -- 172.21.15.90:0/2777084924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 msgr2=0x55d3cc41bd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7fb20c011680 tx=0x7fb20c009920 comp rx=0 tx=0).stop 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 -- 172.21.15.90:0/2777084924 shutdown_connections 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc41bd00 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4153a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2777084924 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc406c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.888+0000 7fb227c00b80 1 -- 172.21.15.90:0/2777084924 >> 172.21.15.90:0/2777084924 conn(0x55d3cc3656a0 msgr2=0x55d3cc401720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.889+0000 7fb227c00b80 1 -- 172.21.15.90:0/2777084924 shutdown_connections 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.889+0000 7fb227c00b80 1 -- 172.21.15.90:0/2777084924 wait complete. 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.889+0000 7fb227c00b80 1 -- start start 2024-04-06T14:19:24.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.889+0000 7fb227c00b80 1 Processor -- start 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc406780 0x55d3cc41bc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc414fc0 0x55d3cc419550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc2f3d80 con 0x55d3cc406780 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc2f3a20 con 0x55d3cc407140 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc1db6a0 con 0x55d3cc414fc0 2024-04-06T14:19:24.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb227c00b80 10 client.0 _ll_register_callbacks cb 0x55d3cc2fd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb218fe7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb218fe7700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37150/0 (socket says 172.21.15.90:37150) 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb218fe7700 1 -- 172.21.15.90:0/967078079 learned_addr learned my addr 172.21.15.90:0/967078079 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.890+0000 7fb219fe9700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc414fc0 0x55d3cc419550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 -- 172.21.15.90:0/967078079 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc414fc0 msgr2=0x55d3cc419550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc414fc0 0x55d3cc419550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 -- 172.21.15.90:0/967078079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc406780 msgr2=0x55d3cc41bc20 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc406780 0x55d3cc41bc20 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 -- 172.21.15.90:0/967078079 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2141aca90 con 0x55d3cc407140 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb218fe7700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fb204097aa0 tx=0x7fb20409a990 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5965acc1decc9860 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2040b2070 con 0x55d3cc407140 2024-04-06T14:19:24.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d3cc318380 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb20409b030 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb20409ee60 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 10 client.27503 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fb20c011330 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb204064080 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb20409a520 con 0x55d3cc407140 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 10 client.27503 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 10 client.27503 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 20 client.27503 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc40b5a0 0x55d3cc40d9c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 4 client.27503 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.891+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d3cc350700 con 0x55d3cc40b5a0 2024-04-06T14:19:24.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.892+0000 7fb2197e8700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc40b5a0 0x55d3cc40d9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.892+0000 7fb2197e8700 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc40b5a0 0x55d3cc40d9c0 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=258983cdb832ebb4 server_cookie=58448fd4996d771 in_seq=0 out_seq=0 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.892+0000 7fb1fcff9700 10 client.27503 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb1fcff9700 1 -- 172.21.15.90:0/967078079 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d3cc350700 con 0x55d3cc40b5a0 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb1fcff9700 10 client.27503 handle_command_reply: tid=0 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 client.27503 shutdown 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 msgr2=0x55d3cc418f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7fb204097aa0 tx=0x7fb20409a990 comp rx=0 tx=0).stop 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 shutdown_connections 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc40b5a0 0x55d3cc40d9c0 crc :-1 s=READY pgs=235 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc414fc0 0x55d3cc419550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc406780 0x55d3cc41bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 --2- 172.21.15.90:0/967078079 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc418f60 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 >> 172.21.15.90:0/967078079 conn(0x55d3cc3656a0 msgr2=0x55d3cc405770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb1fcff9700 0 client.27503 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:24.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 shutdown_connections 2024-04-06T14:19:24.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 1 -- 172.21.15.90:0/967078079 wait complete. 2024-04-06T14:19:24.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.893+0000 7fb227c00b80 20 client.27503 trim_cache size 0 max 16384 2024-04-06T14:19:24.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 Processor -- start 2024-04-06T14:19:24.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 -- start start 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 0x55d3cc416b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 0x55d3cc406b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc420830 con 0x55d3cc414fc0 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc4174b0 con 0x55d3cc406780 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb227c00b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc412960 con 0x55d3cc407140 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55158/0 (socket says 172.21.15.90:55158) 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb2197e8700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 0x55d3cc416b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 -- 172.21.15.90:0/1963864306 learned_addr learned my addr 172.21.15.90:0/1963864306 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 0x55d3cc406b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 -- 172.21.15.90:0/1963864306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 msgr2=0x55d3cc406b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 0x55d3cc406b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 -- 172.21.15.90:0/1963864306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 msgr2=0x55d3cc416b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.911+0000 7fb219fe9700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 0x55d3cc416b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb219fe9700 1 -- 172.21.15.90:0/1963864306 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d3cc185b60 con 0x55d3cc414fc0 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb218fe7700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 0x55d3cc406b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb2197e8700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 0x55d3cc416b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb219fe9700 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fb20c011620 tx=0x7fb20c009b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84ad58400e469623 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb1fffff700 1 -- 172.21.15.90:0/1963864306 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20c00e050 con 0x55d3cc414fc0 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb1fffff700 1 -- 172.21.15.90:0/1963864306 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb20c0042b0 con 0x55d3cc414fc0 2024-04-06T14:19:24.914 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.912+0000 7fb1fffff700 1 -- 172.21.15.90:0/1963864306 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20c0079e0 con 0x55d3cc414fc0 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- 172.21.15.90:0/1963864306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 msgr2=0x55d3cc4202f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 secure :-1 s=READY pgs=843 cs=0 l=1 rev1=1 crypto rx=0x7fb20c011620 tx=0x7fb20c009b70 comp rx=0 tx=0).stop 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- 172.21.15.90:0/1963864306 shutdown_connections 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc4202f0 unknown :-1 s=CLOSED pgs=843 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc406780 0x55d3cc416b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 --2- 172.21.15.90:0/1963864306 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc407140 0x55d3cc406b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- 172.21.15.90:0/1963864306 >> 172.21.15.90:0/1963864306 conn(0x55d3cc3656a0 msgr2=0x55d3cc417830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- 172.21.15.90:0/1963864306 shutdown_connections 2024-04-06T14:19:24.919 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- 172.21.15.90:0/1963864306 wait complete. 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 -- start start 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.913+0000 7fb227c00b80 1 Processor -- start 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc412fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc413500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc4141c0 con 0x55d3cc414fc0 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc414330 con 0x55d3cc407140 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d3cc4144f0 con 0x55d3cc406780 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 10 client.0 _ll_register_callbacks cb 0x55d3cc2fd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55160/0 (socket says 172.21.15.90:55160) 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 -- 172.21.15.90:0/2370780749 learned_addr learned my addr 172.21.15.90:0/2370780749 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:24.920 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb2197e8700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc412fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb218fe7700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc413500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 -- 172.21.15.90:0/2370780749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 msgr2=0x55d3cc412fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc412fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 -- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 msgr2=0x55d3cc413500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc413500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 -- 172.21.15.90:0/2370780749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb210019050 con 0x55d3cc414fc0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb219fe9700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7fb20c00c040 tx=0x7fb20c003fa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f78b523ad95dc61c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb1fcff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb20c007640 con 0x55d3cc414fc0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb20c0077a0 con 0x55d3cc414fc0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb20c00f040 con 0x55d3cc414fc0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.914+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb20c00e020 con 0x55d3cc414fc0 2024-04-06T14:19:24.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 10 client.17778 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d3cc1d93b0 con 0x55d3cc414fc0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb20c00f040 con 0x55d3cc414fc0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb20c005740 con 0x55d3cc414fc0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 10 client.17778 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 10 client.17778 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 20 client.17778 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc4752d0 0x55d3cc4776f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 4 client.17778 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d3cc1a6cf0 con 0x55d3cc4752d0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb2197e8700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc4752d0 0x55d3cc4776f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb1fcff9700 10 client.17778 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.915+0000 7fb2197e8700 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc4752d0 0x55d3cc4776f0 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=188d2372d1f07314 server_cookie=f8dd232918eebb4 in_seq=0 out_seq=0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb1fcff9700 1 -- 172.21.15.90:0/2370780749 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d3cc1a6cf0 con 0x55d3cc4752d0 2024-04-06T14:19:24.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb1fcff9700 10 client.17778 handle_command_reply: tid=0 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 client.17778 shutdown 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 msgr2=0x55d3cc413a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 secure :-1 s=READY pgs=844 cs=0 l=1 rev1=1 crypto rx=0x7fb20c00c040 tx=0x7fb20c003fa0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 shutdown_connections 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d3cc414fc0 0x55d3cc413a60 unknown :-1 s=CLOSED pgs=844 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d3cc4752d0 0x55d3cc4776f0 crc :-1 s=READY pgs=236 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d3cc406780 0x55d3cc412fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb1fcff9700 0 client.17778 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 --2- 172.21.15.90:0/2370780749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d3cc407140 0x55d3cc413500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 >> 172.21.15.90:0/2370780749 conn(0x55d3cc3656a0 msgr2=0x55d3cc4187c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 shutdown_connections 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 1 -- 172.21.15.90:0/2370780749 wait complete. 2024-04-06T14:19:24.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb227c00b80 20 client.17778 trim_cache size 0 max 16384 2024-04-06T14:19:24.924 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:24.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 msgr2=0x7fb2000a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 0x7fb2000a3750 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fb210009e40 tx=0x7fb210009980 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 msgr2=0x7fb214072a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb214072a90 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fb204000c00 tx=0x7fb204004730 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 shutdown_connections 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2000a1290 0x7fb2000a3750 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb21406c750 0x7fb21406cc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.916+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb214069210 0x7fb214072550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.917+0000 7fb21ba4c700 1 --2- 172.21.15.90:0/1014157431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb2141060b0 0x7fb214072a90 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.917+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 >> 172.21.15.90:0/1014157431 conn(0x7fb2140769b0 msgr2=0x7fb2140fe6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.917+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 shutdown_connections 2024-04-06T14:19:24.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:24.917+0000 7fb21ba4c700 1 -- 172.21.15.90:0/1014157431 wait complete. 2024-04-06T14:19:25.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:25 smithi105 ceph-mon[46969]: pgmap v439: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:25 smithi090 ceph-mon[43420]: pgmap v439: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:25.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:25 smithi134 ceph-mon[47549]: pgmap v439: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:25.947 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:19:25.949 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:19:25.950 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 -- 172.21.15.90:0/1468118449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe3010d600 msgr2=0x7efe3010f9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 --2- 172.21.15.90:0/1468118449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe3010d600 0x7efe3010f9f0 secure :-1 s=READY pgs=845 cs=0 l=1 rev1=1 crypto rx=0x7efe2800b0a0 tx=0x7efe2800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 -- 172.21.15.90:0/1468118449 shutdown_connections 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 --2- 172.21.15.90:0/1468118449 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe3010d600 0x7efe3010f9f0 unknown :-1 s=CLOSED pgs=845 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 --2- 172.21.15.90:0/1468118449 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30101c30 0x7efe30068fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 --2- 172.21.15.90:0/1468118449 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe30069500 0x7efe30069980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.208 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 -- 172.21.15.90:0/1468118449 >> 172.21.15.90:0/1468118449 conn(0x7efe300769b0 msgr2=0x7efe30076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 -- 172.21.15.90:0/1468118449 shutdown_connections 2024-04-06T14:19:26.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.207+0000 7efe36141700 1 -- 172.21.15.90:0/1468118449 wait complete. 2024-04-06T14:19:26.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 Processor -- start 2024-04-06T14:19:26.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 -- start start 2024-04-06T14:19:26.209 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe30101c30 0x7efe30199a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe3010d600 0x7efe3019ddc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe3019e400 con 0x7efe30101c30 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe3019e570 con 0x7efe30069500 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe36141700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe3019e730 con 0x7efe3010d600 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe2effd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe30101c30 0x7efe30199a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37184/0 (socket says 172.21.15.90:37184) 2024-04-06T14:19:26.210 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.208+0000 7efe2f7fe700 1 -- 172.21.15.90:0/1930782903 learned_addr learned my addr 172.21.15.90:0/1930782903 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.211 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2ffff700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe3010d600 0x7efe3019ddc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2f7fe700 1 -- 172.21.15.90:0/1930782903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe3010d600 msgr2=0x7efe3019ddc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe3010d600 0x7efe3019ddc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2f7fe700 1 -- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe30101c30 msgr2=0x7efe30199a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe30101c30 0x7efe30199a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.209+0000 7efe2f7fe700 1 -- 172.21.15.90:0/1930782903 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe24009520 con 0x7efe30069500 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7efe2000d950 tx=0x7efe2000dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe2cff9700 1 -- 172.21.15.90:0/1930782903 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe2000f890 con 0x7efe30069500 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe2cff9700 1 -- 172.21.15.90:0/1930782903 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe20010460 con 0x7efe30069500 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe2cff9700 1 -- 172.21.15.90:0/1930782903 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe2000e610 con 0x7efe30069500 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe28009d00 con 0x7efe30069500 2024-04-06T14:19:26.212 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.210+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe3006a7d0 con 0x7efe30069500 2024-04-06T14:19:26.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.214+0000 7efe2cff9700 1 -- 172.21.15.90:0/1930782903 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7efe2000f9f0 con 0x7efe30069500 2024-04-06T14:19:26.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.215+0000 7efe2cff9700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 0x7efe180b47e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.215 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.215+0000 7efe2cff9700 1 -- 172.21.15.90:0/1930782903 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7efe200cc920 con 0x7efe30069500 2024-04-06T14:19:26.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe2effd700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 0x7efe180b47e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe422f5b80 1 Processor -- start 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe2effd700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 0x7efe180b47e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7efe3019aa30 tx=0x7efe24009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe422f5b80 1 -- start start 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.216+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec5932920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec5935250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe422f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5880a10 con 0x557ec5932e60 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe422f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec587e3d0 con 0x557ec5928800 2024-04-06T14:19:26.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe422f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5879cc0 con 0x557ec5929120 2024-04-06T14:19:26.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37200/0 (socket says 172.21.15.90:37200) 2024-04-06T14:19:26.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.217+0000 7efe2f7fe700 1 -- 172.21.15.90:0/941180805 learned_addr learned my addr 172.21.15.90:0/941180805 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2ffff700 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec5935250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2effd700 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec5932920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 -- 172.21.15.90:0/941180805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 msgr2=0x557ec5932920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec5932920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 -- 172.21.15.90:0/941180805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 msgr2=0x557ec5935250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec5935250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 -- 172.21.15.90:0/941180805 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557ec56fb430 con 0x557ec5928800 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7efe20096480 tx=0x7efe20099480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c2c5eb6c94b1eb24 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe15ffb700 1 -- 172.21.15.90:0/941180805 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe2000fca0 con 0x557ec5928800 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.218+0000 7efe15ffb700 1 -- 172.21.15.90:0/941180805 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe2000fe00 con 0x557ec5928800 2024-04-06T14:19:26.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.219+0000 7efe15ffb700 1 -- 172.21.15.90:0/941180805 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe200a1a00 con 0x557ec5928800 2024-04-06T14:19:26.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.219+0000 7efe422f5b80 1 -- 172.21.15.90:0/941180805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 msgr2=0x557ec5928be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.219+0000 7efe422f5b80 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7efe20096480 tx=0x7efe20099480 comp rx=0 tx=0).stop 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 -- 172.21.15.90:0/941180805 shutdown_connections 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec5935250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec5928be0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 --2- 172.21.15.90:0/941180805 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec5932920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 -- 172.21.15.90:0/941180805 >> 172.21.15.90:0/941180805 conn(0x557ec5887720 msgr2=0x557ec5923b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 -- 172.21.15.90:0/941180805 shutdown_connections 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 -- 172.21.15.90:0/941180805 wait complete. 2024-04-06T14:19:26.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.220+0000 7efe422f5b80 1 -- start start 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 Processor -- start 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec59328c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec592fbb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5815e00 con 0x557ec5932e60 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5815aa0 con 0x557ec5928800 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec56fd720 con 0x557ec5929120 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe422f5b80 10 client.0 _ll_register_callbacks cb 0x557ec581f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe2effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec592fbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe2ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe2ffff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55206/0 (socket says 172.21.15.90:55206) 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.221+0000 7efe2ffff700 1 -- 172.21.15.90:0/1348145424 learned_addr learned my addr 172.21.15.90:0/1348145424 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec59328c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 -- 172.21.15.90:0/1348145424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 msgr2=0x557ec592fbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec592fbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 -- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 msgr2=0x557ec59328c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec59328c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 -- 172.21.15.90:0/1348145424 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe3006a7d0 con 0x557ec5932e60 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe2ffff700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7efe301020e0 tx=0x7efe2800bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3ed6aa99035a5efb server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe0effd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe2800e040 con 0x557ec5932e60 2024-04-06T14:19:26.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7efe28003e10 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557ec583a400 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7efe28009b80 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe422f5b80 10 client.17790 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x557ec56a7be0 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.222+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557ec583a400 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7efe28022020 con 0x557ec5932e60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 10 client.17790 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 10 client.17790 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 20 client.17790 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec5942a60 0x557ec5944e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 4 client.17790 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557ec5872780 con 0x557ec5942a60 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec5942a60 0x557ec5944e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.227 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe0effd700 10 client.17790 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.223+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec5942a60 0x557ec5944e80 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=24dfe3893a2a9d9 server_cookie=e6a7209db203c4c1 in_seq=0 out_seq=0 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe0effd700 1 -- 172.21.15.90:0/1348145424 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x557ec5872780 con 0x557ec5942a60 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe0effd700 10 client.17790 handle_command_reply: tid=0 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 client.17790 shutdown 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 msgr2=0x557ec59302f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 secure :-1 s=READY pgs=846 cs=0 l=1 rev1=1 crypto rx=0x7efe301020e0 tx=0x7efe2800bce0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 shutdown_connections 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5932e60 0x557ec59302f0 unknown :-1 s=CLOSED pgs=846 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5928800 0x557ec59328c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec5942a60 0x557ec5944e80 crc :-1 s=READY pgs=237 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 --2- 172.21.15.90:0/1348145424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5929120 0x557ec592fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 >> 172.21.15.90:0/1348145424 conn(0x557ec5887720 msgr2=0x557ec5922ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe0effd700 0 client.17790 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 shutdown_connections 2024-04-06T14:19:26.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.224+0000 7efe422f5b80 1 -- 172.21.15.90:0/1348145424 wait complete. 2024-04-06T14:19:26.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.225+0000 7efe422f5b80 20 client.17790 trim_cache size 0 max 16384 2024-04-06T14:19:26.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 Processor -- start 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 -- start start 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec59323f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec592ce10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec59311b0 con 0x557ec5929120 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec592a9b0 con 0x557ec5932e60 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe422f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5939ff0 con 0x557ec5928800 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe2f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48838/0 (socket says 172.21.15.90:48838) 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe2f7fe700 1 -- 172.21.15.90:0/2481677507 learned_addr learned my addr 172.21.15.90:0/2481677507 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.247+0000 7efe2effd700 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec59323f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2ffff700 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec592ce10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 -- 172.21.15.90:0/2481677507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 msgr2=0x557ec592ce10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec592ce10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.249 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 -- 172.21.15.90:0/2481677507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 msgr2=0x557ec59323f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec59323f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 -- 172.21.15.90:0/2481677507 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557ec591fbe0 con 0x557ec5928800 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7efe20099710 tx=0x7efe20010f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=17dfc41268047efd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe167fc700 1 -- 172.21.15.90:0/2481677507 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe200b2170 con 0x557ec5928800 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe167fc700 1 -- 172.21.15.90:0/2481677507 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe200b1070 con 0x557ec5928800 2024-04-06T14:19:26.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.248+0000 7efe167fc700 1 -- 172.21.15.90:0/2481677507 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe20098f00 con 0x557ec5928800 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- 172.21.15.90:0/2481677507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 msgr2=0x557ec5939ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7efe20099710 tx=0x7efe20010f90 comp rx=0 tx=0).stop 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- 172.21.15.90:0/2481677507 shutdown_connections 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec592ce10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5939ab0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 --2- 172.21.15.90:0/2481677507 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec59323f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- 172.21.15.90:0/2481677507 >> 172.21.15.90:0/2481677507 conn(0x557ec5887720 msgr2=0x557ec59497e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- 172.21.15.90:0/2481677507 shutdown_connections 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- 172.21.15.90:0/2481677507 wait complete. 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.249+0000 7efe422f5b80 1 -- start start 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 Processor -- start 2024-04-06T14:19:26.252 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5999730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec5999c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5939240 con 0x557ec5929120 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec59393b0 con 0x557ec5932e60 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ec5939570 con 0x557ec5928800 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.250+0000 7efe422f5b80 10 client.0 _ll_register_callbacks cb 0x557ec581f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37230/0 (socket says 172.21.15.90:37230) 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 -- 172.21.15.90:0/3612985165 learned_addr learned my addr 172.21.15.90:0/3612985165 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5999730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 -- 172.21.15.90:0/3612985165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 msgr2=0x557ec5999730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5999730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 -- 172.21.15.90:0/3612985165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 msgr2=0x557ec5999c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.253 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec5999c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 -- 172.21.15.90:0/3612985165 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557ec56fb430 con 0x557ec5932e60 2024-04-06T14:19:26.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe2ffff700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7efe2800b0a0 tx=0x7efe2800b7b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f25e89a1efc57993 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.254 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.251+0000 7efe0effd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe2800e040 con 0x557ec5932e60 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7efe280076b0 con 0x557ec5932e60 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7efe2000fe00 con 0x557ec5932e60 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7efe28007870 con 0x557ec5932e60 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe422f5b80 10 client.27527 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.252+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x557ec583ff20 con 0x557ec5932e60 2024-04-06T14:19:26.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.253+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe2000fe00 con 0x557ec5932e60 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.253+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7efe28019040 con 0x557ec5932e60 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 10 client.27527 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 10 client.27527 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 20 client.27527 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec598d620 0x557ec598fa40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 4 client.27527 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x557ec56c8d70 con 0x557ec598d620 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec598d620 0x557ec598fa40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe0effd700 10 client.27527 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.254+0000 7efe2f7fe700 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec598d620 0x557ec598fa40 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=30090d4f846e3c0d server_cookie=7065810b6e61eb5c in_seq=0 out_seq=0 2024-04-06T14:19:26.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.255+0000 7efe0effd700 1 -- 172.21.15.90:0/3612985165 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x557ec56c8d70 con 0x557ec598d620 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.255+0000 7efe0effd700 10 client.27527 handle_command_reply: tid=0 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.255+0000 7efe422f5b80 1 client.27527 shutdown 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 msgr2=0x557ec5938c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7efe2800b0a0 tx=0x7efe2800b7b0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 shutdown_connections 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x557ec5932e60 0x557ec5938c60 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x557ec598d620 0x557ec598fa40 crc :-1 s=READY pgs=238 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x557ec5928800 0x557ec5999730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 --2- 172.21.15.90:0/3612985165 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x557ec5929120 0x557ec5999c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe0effd700 0 client.27527 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 >> 172.21.15.90:0/3612985165 conn(0x557ec5887720 msgr2=0x557ec592e480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 shutdown_connections 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 1 -- 172.21.15.90:0/3612985165 wait complete. 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe422f5b80 20 client.27527 trim_cache size 0 max 16384 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "38f6a89f-cf5c-4a42-9e13-dc747b66f5d8", 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:19:26.259 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 msgr2=0x7efe180b47e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 0x7efe180b47e0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7efe3019aa30 tx=0x7efe24009260 comp rx=0 tx=0).stop 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 msgr2=0x7efe301994e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.256+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7efe2000d950 tx=0x7efe2000dc60 comp rx=0 tx=0).stop 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 shutdown_connections 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe180b2320 0x7efe180b47e0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe3010d600 0x7efe3019ddc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe30069500 0x7efe301994e0 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 --2- 172.21.15.90:0/1930782903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe30101c30 0x7efe30199a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 >> 172.21.15.90:0/1930782903 conn(0x7efe300769b0 msgr2=0x7efe300feef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 shutdown_connections 2024-04-06T14:19:26.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.258+0000 7efe36141700 1 -- 172.21.15.90:0/1930782903 wait complete. 2024-04-06T14:19:26.275 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:19:26.275 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:19:26.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.470+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/3500116367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 msgr2=0x7fcdac10dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.470+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/3500116367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac10dc30 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7fcda400b0a0 tx=0x7fcda400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/3500116367 shutdown_connections 2024-04-06T14:19:26.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/3500116367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac10dc30 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/3500116367 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac100f90 0x7fcdac101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/3500116367 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac101940 0x7fcdac105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/3500116367 >> 172.21.15.90:0/3500116367 conn(0x7fcdac0fc7f0 msgr2=0x7fcdac0fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/3500116367 shutdown_connections 2024-04-06T14:19:26.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/3500116367 wait complete. 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 Processor -- start 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.471+0000 7fcdb2cdb700 1 -- start start 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 0x7fcdac19bcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac19c270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdac195a50 con 0x7fcdac194e10 2024-04-06T14:19:26.474 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdac195bc0 con 0x7fcdac1060b0 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb2cdb700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdac199880 con 0x7fcdac100f90 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 0x7fcdac19bcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdabfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac19c270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 0x7fcdac19bcf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48882/0 (socket says 172.21.15.90:48882) 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55254/0 (socket says 172.21.15.90:55254) 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb0a77700 1 -- 172.21.15.90:0/1447545061 learned_addr learned my addr 172.21.15.90:0/1447545061 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 -- 172.21.15.90:0/1447545061 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 msgr2=0x7fcdac19bcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 0x7fcdac19bcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 -- 172.21.15.90:0/1447545061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 msgr2=0x7fcdac19c270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac19c270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.475 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 -- 172.21.15.90:0/1447545061 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcda4009d00 con 0x7fcdac194e10 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.472+0000 7fcdb1278700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 secure :-1 s=READY pgs=847 cs=0 l=1 rev1=1 crypto rx=0x7fcda400b070 tx=0x7fcda4009c60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.473+0000 7fcda9ffb700 1 -- 172.21.15.90:0/1447545061 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda400e040 con 0x7fcdac194e10 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.473+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcdac199ab0 con 0x7fcdac194e10 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.473+0000 7fcda9ffb700 1 -- 172.21.15.90:0/1447545061 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcda4007990 con 0x7fcdac194e10 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.473+0000 7fcda9ffb700 1 -- 172.21.15.90:0/1447545061 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda4005040 con 0x7fcdac194e10 2024-04-06T14:19:26.476 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.473+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcdac19a010 con 0x7fcdac194e10 2024-04-06T14:19:26.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.478+0000 7fcda9ffb700 1 -- 172.21.15.90:0/1447545061 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fcda40051a0 con 0x7fcdac194e10 2024-04-06T14:19:26.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcda9ffb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 0x7fcd9c0a37b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcda9ffb700 1 -- 172.21.15.90:0/1447545061 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcda40ce230 con 0x7fcdac194e10 2024-04-06T14:19:26.479 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 0x7fcd9c0a37b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 0x7fcd9c0a37b0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fcda0009d20 tx=0x7fcda0009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcdbee8fb80 1 Processor -- start 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.479+0000 7fcdbee8fb80 1 -- start start 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fdab60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.481 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fe71d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1f32990 con 0x5590c1fe4de0 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1f30350 con 0x5590c1fdb0a0 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1f2bc40 con 0x5590c1fda780 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdabfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdabfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37270/0 (socket says 172.21.15.90:37270) 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdabfff700 1 -- 172.21.15.90:0/2509033537 learned_addr learned my addr 172.21.15.90:0/2509033537 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fdab60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.480+0000 7fcdb1278700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fe71d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 -- 172.21.15.90:0/2509033537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 msgr2=0x5590c1fdab60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fdab60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 -- 172.21.15.90:0/2509033537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 msgr2=0x5590c1fe48a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.482 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 -- 172.21.15.90:0/2509033537 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5590c1dad3b0 con 0x5590c1fe4de0 2024-04-06T14:19:26.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdabfff700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:26.483 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.481+0000 7fcdb1278700 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fe71d0 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cb960 tx=0x7fcda400b3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a229ad50b3befe4c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.482+0000 7fcd96ffd700 1 -- 172.21.15.90:0/2509033537 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda409f6c0 con 0x5590c1fe4de0 2024-04-06T14:19:26.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.482+0000 7fcd96ffd700 1 -- 172.21.15.90:0/2509033537 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcda40a3c80 con 0x5590c1fe4de0 2024-04-06T14:19:26.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.482+0000 7fcd96ffd700 1 -- 172.21.15.90:0/2509033537 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda401d470 con 0x5590c1fe4de0 2024-04-06T14:19:26.484 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/2509033537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 msgr2=0x5590c1fe71d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fe71d0 secure :-1 s=READY pgs=848 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cb960 tx=0x7fcda400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/2509033537 shutdown_connections 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fe71d0 unknown :-1 s=CLOSED pgs=848 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fdab60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/2509033537 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe48a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/2509033537 >> 172.21.15.90:0/2509033537 conn(0x5590c1f396a0 msgr2=0x5590c1fd5a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/2509033537 shutdown_connections 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/2509033537 wait complete. 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.483+0000 7fcdbee8fb80 1 -- start start 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 Processor -- start 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fe4de0 0x5590c1fe25d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1ec7d80 con 0x5590c1fdb0a0 2024-04-06T14:19:26.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1ec7a20 con 0x5590c1fe4de0 2024-04-06T14:19:26.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1daf6a0 con 0x5590c1fda780 2024-04-06T14:19:26.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdbee8fb80 10 client.0 _ll_register_callbacks cb 0x5590c1ed1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdabfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdabfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55274/0 (socket says 172.21.15.90:55274) 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48890/0 (socket says 172.21.15.90:48890) 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.484+0000 7fcdabfff700 1 -- 172.21.15.90:0/3841738711 learned_addr learned my addr 172.21.15.90:0/3841738711 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb1278700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fe4de0 0x5590c1fe25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 -- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fe4de0 msgr2=0x5590c1fe25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fe4de0 0x5590c1fe25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 -- 172.21.15.90:0/3841738711 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 msgr2=0x5590c1fe2090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 -- 172.21.15.90:0/3841738711 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcd980096b0 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdabfff700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fcda001ec20 tx=0x7fcda00047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e4dd80b980e9df04 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcd8bfff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda001e220 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5590c1eec380 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcda000ea50 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.485+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcda000ec10 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 10 client.27268 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7fcdac19a010 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5590c1eec380 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcda00263f0 con 0x5590c1fda780 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 10 client.27268 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:26.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 10 client.27268 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:26.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 20 client.27268 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:26.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c1ff4880 0x5590c1ff6ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 4 client.27268 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:26.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5590c1f24700 con 0x5590c1ff4880 2024-04-06T14:19:26.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.486+0000 7fcdabfff700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c1ff4880 0x5590c1ff6ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.488+0000 7fcd8bfff700 10 client.27268 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.488+0000 7fcdabfff700 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c1ff4880 0x5590c1ff6ca0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7397ba240e1df938 server_cookie=8eaf3689e790bfcb in_seq=0 out_seq=0 2024-04-06T14:19:26.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3841738711 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5590c1f24700 con 0x5590c1ff4880 2024-04-06T14:19:26.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcd8bfff700 10 client.27268 handle_command_reply: tid=0 2024-04-06T14:19:26.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 client.27268 shutdown 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 msgr2=0x5590c1fe1b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fcda001ec20 tx=0x7fcda00047c0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 shutdown_connections 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fe4de0 0x5590c1fe25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c1ff4880 0x5590c1ff6ca0 crc :-1 s=READY pgs=239 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1fe1b50 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3841738711 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe2090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcd8bfff700 0 client.27268 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.489+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 >> 172.21.15.90:0/3841738711 conn(0x5590c1f396a0 msgr2=0x5590c1fd4c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.490+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 shutdown_connections 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.490+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3841738711 wait complete. 2024-04-06T14:19:26.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.490+0000 7fcdbee8fb80 20 client.27268 trim_cache size 0 max 16384 2024-04-06T14:19:26.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 Processor -- start 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 -- start start 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fda780 0x5590c1fd5520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fdb0a0 0x5590c1fd5a60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1fe31c0 con 0x5590c1fe4de0 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1fd67c0 con 0x5590c1fda780 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1fd5fa0 con 0x5590c1fdb0a0 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdb1278700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdb1278700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:55276/0 (socket says 172.21.15.90:55276) 2024-04-06T14:19:26.514 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.513+0000 7fcdb1278700 1 -- 172.21.15.90:0/959974104 learned_addr learned my addr 172.21.15.90:0/959974104 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.514+0000 7fcdb1278700 1 -- 172.21.15.90:0/959974104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fdb0a0 msgr2=0x5590c1fd5a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.514+0000 7fcdb1278700 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fdb0a0 0x5590c1fd5a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.514+0000 7fcdb1278700 1 -- 172.21.15.90:0/959974104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fda780 msgr2=0x5590c1fd5520 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:19:26.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.514+0000 7fcdb1278700 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fda780 0x5590c1fd5520 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.515 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.514+0000 7fcdb1278700 1 -- 172.21.15.90:0/959974104 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5590c1d59b60 con 0x5590c1fe4de0 2024-04-06T14:19:26.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.515+0000 7fcdb1278700 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cbf60 tx=0x7fcda40099e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=579caafffecbce77 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.516+0000 7fcd977fe700 1 -- 172.21.15.90:0/959974104 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda409a350 con 0x5590c1fe4de0 2024-04-06T14:19:26.516 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.516+0000 7fcd977fe700 1 -- 172.21.15.90:0/959974104 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcda409f6c0 con 0x5590c1fe4de0 2024-04-06T14:19:26.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.516+0000 7fcd977fe700 1 -- 172.21.15.90:0/959974104 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda4011c20 con 0x5590c1fe4de0 2024-04-06T14:19:26.517 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/959974104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 msgr2=0x5590c1fdee10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 secure :-1 s=READY pgs=849 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cbf60 tx=0x7fcda40099e0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/959974104 shutdown_connections 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fdee10 unknown :-1 s=CLOSED pgs=849 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fda780 0x5590c1fd5520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/959974104 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fdb0a0 0x5590c1fd5a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/959974104 >> 172.21.15.90:0/959974104 conn(0x5590c1f396a0 msgr2=0x5590c1fe0460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/959974104 shutdown_connections 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/959974104 wait complete. 2024-04-06T14:19:26.518 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.517+0000 7fcdbee8fb80 1 -- start start 2024-04-06T14:19:26.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 Processor -- start 2024-04-06T14:19:26.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1feba20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe9ef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.521 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fea630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c1fead40 con 0x5590c1fe4de0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c2048680 con 0x5590c1fdb0a0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5590c20487c0 con 0x5590c1fda780 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdbee8fb80 10 client.0 _ll_register_callbacks cb 0x5590c1ed1470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1feba20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdb0a77700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1feba20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:48912/0 (socket says 172.21.15.90:48912) 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdb0a77700 1 -- 172.21.15.90:0/3475007365 learned_addr learned my addr 172.21.15.90:0/3475007365 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdabfff700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe9ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.518+0000 7fcdb1278700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fea630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 -- 172.21.15.90:0/3475007365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 msgr2=0x5590c1feba20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1feba20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.522 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 -- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 msgr2=0x5590c1fe9ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe9ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 -- 172.21.15.90:0/3475007365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5590c1dad3b0 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdabfff700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe9ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdb1278700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fea630 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cbee0 tx=0x7fcda40097e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e94044094f6806fe server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcd8bfff700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcda401d670 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fcda4011c20 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.519+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fcda40a3c80 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcda4011de0 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 10 client.17811 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5590c1ef1ea0 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcda40a3c80 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fcda40a37f0 con 0x5590c1fe4de0 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 10 client.17811 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:26.523 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 10 client.17811 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 20 client.17811 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c2049a20 0x5590c204be10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 4 client.17811 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.520+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5590c1d7acf0 con 0x5590c2049a20 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.521+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c2049a20 0x5590c204be10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.521+0000 7fcd8bfff700 10 client.17811 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.521+0000 7fcdb0a77700 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c2049a20 0x5590c204be10 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c745f0ec648406c2 server_cookie=2622f8086b08b660 in_seq=0 out_seq=0 2024-04-06T14:19:26.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcd8bfff700 1 -- 172.21.15.90:0/3475007365 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x5590c1d7acf0 con 0x5590c2049a20 2024-04-06T14:19:26.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcd8bfff700 10 client.17811 handle_command_reply: tid=0 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 client.17811 shutdown 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 msgr2=0x5590c1fea630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fea630 secure :-1 s=READY pgs=850 cs=0 l=1 rev1=1 crypto rx=0x7fcda40cbee0 tx=0x7fcda40097e0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 shutdown_connections 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5590c1fe4de0 0x5590c1fea630 unknown :-1 s=CLOSED pgs=850 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5590c2049a20 0x5590c204be10 crc :-1 s=READY pgs=240 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5590c1fda780 0x5590c1feba20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 --2- 172.21.15.90:0/3475007365 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5590c1fdb0a0 0x5590c1fe9ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 >> 172.21.15.90:0/3475007365 conn(0x5590c1f396a0 msgr2=0x5590c1fe0390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcd8bfff700 0 client.17811 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 shutdown_connections 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 1 -- 172.21.15.90:0/3475007365 wait complete. 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.522+0000 7fcdbee8fb80 20 client.17811 trim_cache size 0 max 16384 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (187 inodes in the stack)", 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stdout: "38f6a89f-cf5c-4a42-9e13-dc747b66f5d8": { 2024-04-06T14:19:26.527 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "38f6a89f-cf5c-4a42-9e13-dc747b66f5d8", 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:26.528 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 msgr2=0x7fcd9c0a37b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 0x7fcd9c0a37b0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7fcda0009d20 tx=0x7fcda0009260 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 msgr2=0x7fcdac1952c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 secure :-1 s=READY pgs=847 cs=0 l=1 rev1=1 crypto rx=0x7fcda400b070 tx=0x7fcda4009c60 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 shutdown_connections 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fcd9c0a12f0 0x7fcd9c0a37b0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fcdac194e10 0x7fcdac1952c0 unknown :-1 s=CLOSED pgs=847 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fcdac100f90 0x7fcdac19bcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 --2- 172.21.15.90:0/1447545061 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcdac1060b0 0x7fcdac19c270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 >> 172.21.15.90:0/1447545061 conn(0x7fcdac0fc7f0 msgr2=0x7fcdac0feaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 shutdown_connections 2024-04-06T14:19:26.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:26.523+0000 7fcdb2cdb700 1 -- 172.21.15.90:0/1447545061 wait complete. 2024-04-06T14:19:26.540 INFO:tasks.cephfs.filesystem:scrub status for tag:38f6a89f-cf5c-4a42-9e13-dc747b66f5d8 - {'path': '/', 'tag': '38f6a89f-cf5c-4a42-9e13-dc747b66f5d8', 'options': 'recursive,force'} 2024-04-06T14:19:27.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:27 smithi105 ceph-mon[46969]: pgmap v440: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:27.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:27 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:19:27.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:27 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:27.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:27 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:19:27.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:27 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:27 smithi090 ceph-mon[43420]: pgmap v440: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:27 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:19:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:27 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:27 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:19:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:27 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:27 smithi134 ceph-mon[47549]: pgmap v440: 97 pgs: 97 active+clean; 5.6 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 11 MiB/s wr, 620 op/s 2024-04-06T14:19:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:27 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:19:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:27 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:27 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:19:27.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:27 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:19:28.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 820.0 s, 1307.8 tps, lat 76.528 ms stddev 83.344 2024-04-06T14:19:29.396 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:29 smithi134 ceph-mon[47549]: pgmap v441: 97 pgs: 97 active+clean; 5.7 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 9.1 MiB/s wr, 592 op/s 2024-04-06T14:19:29.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:29 smithi105 ceph-mon[46969]: pgmap v441: 97 pgs: 97 active+clean; 5.7 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 9.1 MiB/s wr, 592 op/s 2024-04-06T14:19:29.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:29 smithi090 ceph-mon[43420]: pgmap v441: 97 pgs: 97 active+clean; 5.7 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 9.1 MiB/s wr, 592 op/s 2024-04-06T14:19:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:31 smithi105 ceph-mon[46969]: pgmap v442: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.4 MiB/s wr, 559 op/s 2024-04-06T14:19:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:31 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:19:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:31 smithi090 ceph-mon[43420]: pgmap v442: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.4 MiB/s wr, 559 op/s 2024-04-06T14:19:31.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:31 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:19:31.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:31 smithi134 ceph-mon[47549]: pgmap v442: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 8.4 MiB/s wr, 559 op/s 2024-04-06T14:19:31.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:31 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:19:33.031 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 825.0 s, 1654.2 tps, lat 59.941 ms stddev 69.643 2024-04-06T14:19:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:33 smithi105 ceph-mon[46969]: pgmap v443: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 670 op/s 2024-04-06T14:19:33.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:33 smithi090 ceph-mon[43420]: pgmap v443: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 670 op/s 2024-04-06T14:19:33.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:33 smithi134 ceph-mon[47549]: pgmap v443: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 10 MiB/s wr, 670 op/s 2024-04-06T14:19:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:35 smithi105 ceph-mon[46969]: pgmap v444: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 12 MiB/s wr, 780 op/s 2024-04-06T14:19:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:35 smithi090 ceph-mon[43420]: pgmap v444: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 12 MiB/s wr, 780 op/s 2024-04-06T14:19:35.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:35 smithi134 ceph-mon[47549]: pgmap v444: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 81 KiB/s rd, 12 MiB/s wr, 780 op/s 2024-04-06T14:19:37.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:37 smithi105 ceph-mon[46969]: pgmap v445: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 9.4 MiB/s wr, 648 op/s 2024-04-06T14:19:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:37 smithi090 ceph-mon[43420]: pgmap v445: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 9.4 MiB/s wr, 648 op/s 2024-04-06T14:19:37.648 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:37 smithi134 ceph-mon[47549]: pgmap v445: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 9.4 MiB/s wr, 648 op/s 2024-04-06T14:19:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 830.0 s, 1768.7 tps, lat 57.008 ms stddev 69.033 2024-04-06T14:19:39.293 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:39 smithi090 ceph-mon[43420]: pgmap v446: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 13 MiB/s wr, 866 op/s 2024-04-06T14:19:39.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:39 smithi134 ceph-mon[47549]: pgmap v446: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 13 MiB/s wr, 866 op/s 2024-04-06T14:19:39.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:39 smithi105 ceph-mon[46969]: pgmap v446: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 74 KiB/s rd, 13 MiB/s wr, 866 op/s 2024-04-06T14:19:41.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:41 smithi105 ceph-mon[46969]: pgmap v447: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 9.9 MiB/s wr, 693 op/s 2024-04-06T14:19:41.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:41 smithi090 ceph-mon[43420]: pgmap v447: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 9.9 MiB/s wr, 693 op/s 2024-04-06T14:19:41.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:41 smithi134 ceph-mon[47549]: pgmap v447: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 9.9 MiB/s wr, 693 op/s 2024-04-06T14:19:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 835.0 s, 809.4 tps, lat 120.885 ms stddev 202.068 2024-04-06T14:19:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: pgmap v448: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 16 MiB/s wr, 1.10k op/s 2024-04-06T14:19:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:19:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:43 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: pgmap v448: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 16 MiB/s wr, 1.10k op/s 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:43 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: pgmap v448: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 38 KiB/s rd, 16 MiB/s wr, 1.10k op/s 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:43.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:43 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:44.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:44 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:19:44.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:44 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:19:44.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:44 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:44.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:44 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:19:44.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:44 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:19:44.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:44 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:44.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:44 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:19:44.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:44 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:19:44.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:44 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:19:45.647 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:45 smithi134 ceph-mon[47549]: pgmap v449: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 25 MiB/s wr, 1.70k op/s 2024-04-06T14:19:45.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:45 smithi105 ceph-mon[46969]: pgmap v449: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 25 MiB/s wr, 1.70k op/s 2024-04-06T14:19:45.801 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:45 smithi090 ceph-mon[43420]: pgmap v449: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 25 MiB/s wr, 1.70k op/s 2024-04-06T14:19:47.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:47 smithi134 ceph-mon[47549]: pgmap v450: 97 pgs: 97 active+clean; 6.0 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.75k op/s 2024-04-06T14:19:47.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:47 smithi105 ceph-mon[46969]: pgmap v450: 97 pgs: 97 active+clean; 6.0 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.75k op/s 2024-04-06T14:19:47.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:47 smithi090 ceph-mon[43420]: pgmap v450: 97 pgs: 97 active+clean; 6.0 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 39 KiB/s rd, 27 MiB/s wr, 1.75k op/s 2024-04-06T14:19:48.030 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 840.0 s, 1035.2 tps, lat 98.154 ms stddev 115.895 2024-04-06T14:19:49.290 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:49 smithi090 ceph-mon[43420]: pgmap v451: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 31 MiB/s wr, 1.98k op/s 2024-04-06T14:19:49.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:49 smithi134 ceph-mon[47549]: pgmap v451: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 31 MiB/s wr, 1.98k op/s 2024-04-06T14:19:49.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:49 smithi105 ceph-mon[46969]: pgmap v451: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 91 KiB/s rd, 31 MiB/s wr, 1.98k op/s 2024-04-06T14:19:51.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:51 smithi105 ceph-mon[46969]: pgmap v452: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 29 MiB/s wr, 1.81k op/s 2024-04-06T14:19:51.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:51 smithi090 ceph-mon[43420]: pgmap v452: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 29 MiB/s wr, 1.81k op/s 2024-04-06T14:19:51.649 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:51 smithi134 ceph-mon[47549]: pgmap v452: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 29 MiB/s wr, 1.81k op/s 2024-04-06T14:19:53.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 845.0 s, 1369.4 tps, lat 73.278 ms stddev 80.693 2024-04-06T14:19:53.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:53 smithi105 ceph-mon[46969]: pgmap v453: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 30 MiB/s wr, 1.87k op/s 2024-04-06T14:19:53.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:53 smithi090 ceph-mon[43420]: pgmap v453: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 30 MiB/s wr, 1.87k op/s 2024-04-06T14:19:53.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:53 smithi134 ceph-mon[47549]: pgmap v453: 97 pgs: 97 active+clean; 6.0 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 77 KiB/s rd, 30 MiB/s wr, 1.87k op/s 2024-04-06T14:19:55.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:55 smithi105 ceph-mon[46969]: pgmap v454: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 26 MiB/s wr, 1.54k op/s 2024-04-06T14:19:55.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:55 smithi090 ceph-mon[43420]: pgmap v454: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 26 MiB/s wr, 1.54k op/s 2024-04-06T14:19:55.646 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:55 smithi134 ceph-mon[47549]: pgmap v454: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 26 MiB/s wr, 1.54k op/s 2024-04-06T14:19:56.541 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:19:56.541 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:19:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.785+0000 7f9b48256700 1 -- 172.21.15.90:0/3026470463 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b400695a0 msgr2=0x7f9b40069a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.785 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.785+0000 7f9b48256700 1 --2- 172.21.15.90:0/3026470463 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b400695a0 0x7f9b40069a20 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f9b30009990 tx=0x7f9b30009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 -- 172.21.15.90:0/3026470463 shutdown_connections 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 --2- 172.21.15.90:0/3026470463 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40103740 0x7f9b40103bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 --2- 172.21.15.90:0/3026470463 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b40105c40 0x7f9b40069060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 --2- 172.21.15.90:0/3026470463 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b400695a0 0x7f9b40069a20 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 -- 172.21.15.90:0/3026470463 >> 172.21.15.90:0/3026470463 conn(0x7f9b40076d40 msgr2=0x7f9b40077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 -- 172.21.15.90:0/3026470463 shutdown_connections 2024-04-06T14:19:56.786 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 -- 172.21.15.90:0/3026470463 wait complete. 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 Processor -- start 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.786+0000 7f9b48256700 1 -- start start 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 0x7f9b401a3510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 0x7f9b401a3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40105c40 0x7f9b4019d640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b4019dca0 con 0x7f9b40105c40 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b4019de10 con 0x7f9b400695a0 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b48256700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b4019dfd0 con 0x7f9b40103740 2024-04-06T14:19:56.788 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 0x7f9b401a3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 0x7f9b401a3a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55052/0 (socket says 172.21.15.90:55052) 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b457f1700 1 -- 172.21.15.90:0/1712952674 learned_addr learned my addr 172.21.15.90:0/1712952674 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 0x7f9b401a3510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.787+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40105c40 0x7f9b4019d640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 -- 172.21.15.90:0/1712952674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 msgr2=0x7f9b401a3a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 0x7f9b401a3a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 -- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40105c40 msgr2=0x7f9b4019d640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40105c40 0x7f9b4019d640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.789 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 -- 172.21.15.90:0/1712952674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b3c009520 con 0x7f9b400695a0 2024-04-06T14:19:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 0x7f9b401a3510 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f9b3c00e9f0 tx=0x7f9b3c00ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b36ffd700 1 -- 172.21.15.90:0/1712952674 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b3c00cb70 con 0x7f9b400695a0 2024-04-06T14:19:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b36ffd700 1 -- 172.21.15.90:0/1712952674 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b3c0044e0 con 0x7f9b400695a0 2024-04-06T14:19:56.790 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.788+0000 7f9b36ffd700 1 -- 172.21.15.90:0/1712952674 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b3c003d80 con 0x7f9b400695a0 2024-04-06T14:19:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.789+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b300095f0 con 0x7f9b400695a0 2024-04-06T14:19:56.791 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.789+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b4006a7d0 con 0x7f9b400695a0 2024-04-06T14:19:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 Processor -- start 2024-04-06T14:19:56.792 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b36ffd700 1 -- 172.21.15.90:0/1712952674 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9b3c003ee0 con 0x7f9b400695a0 2024-04-06T14:19:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 -- start start 2024-04-06T14:19:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36ccdb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd78a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.794 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36c25990 con 0x562f36cd7de0 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36c23350 con 0x562f36cce0a0 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36c1ec40 con 0x562f36ccd780 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.793+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b45ff2700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd78a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60670/0 (socket says 172.21.15.90:60670) 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 -- 172.21.15.90:0/888077065 learned_addr learned my addr 172.21.15.90:0/888077065 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b467f3700 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36ccdb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.795 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b36ffd700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 0x7f9b2c0a3920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 -- 172.21.15.90:0/888077065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 msgr2=0x562f36ccdb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36ccdb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 -- 172.21.15.90:0/888077065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 msgr2=0x562f36cd78a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd78a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b36ffd700 1 -- 172.21.15.90:0/1712952674 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b3c014070 con 0x7f9b400695a0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 -- 172.21.15.90:0/888077065 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f36aa03b0 con 0x562f36cd7de0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 0x7f9b2c0a3920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.794+0000 7f9b457f1700 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f9b300050f0 tx=0x7f9b30005120 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3ba40e2f30af28b5 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b2bfff700 1 -- 172.21.15.90:0/888077065 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b3001d070 con 0x562f36cd7de0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 0x7f9b2c0a3920 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f9b3800a4e0 tx=0x7f9b3800c040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b2bfff700 1 -- 172.21.15.90:0/888077065 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b3001e040 con 0x562f36cd7de0 2024-04-06T14:19:56.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b2bfff700 1 -- 172.21.15.90:0/888077065 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b30004a40 con 0x562f36cd7de0 2024-04-06T14:19:56.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 -- 172.21.15.90:0/888077065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 msgr2=0x562f36cda1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 secure :-1 s=READY pgs=851 cs=0 l=1 rev1=1 crypto rx=0x7f9b300050f0 tx=0x7f9b30005120 comp rx=0 tx=0).stop 2024-04-06T14:19:56.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 -- 172.21.15.90:0/888077065 shutdown_connections 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd7de0 0x562f36cda1d0 unknown :-1 s=CLOSED pgs=851 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36ccdb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/888077065 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd78a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.795+0000 7f9b5440ab80 1 -- 172.21.15.90:0/888077065 >> 172.21.15.90:0/888077065 conn(0x562f36c2c6a0 msgr2=0x562f36cc8a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- 172.21.15.90:0/888077065 shutdown_connections 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- 172.21.15.90:0/888077065 wait complete. 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- start start 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 Processor -- start 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36cd4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd5310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36bbad80 con 0x562f36cd5850 2024-04-06T14:19:56.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36bbaa20 con 0x562f36cce0a0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36aa26a0 con 0x562f36ccd780 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b5440ab80 10 client.0 _ll_register_callbacks cb 0x562f36bc4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.796+0000 7f9b467f3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60672/0 (socket says 172.21.15.90:60672) 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 -- 172.21.15.90:0/1216610690 learned_addr learned my addr 172.21.15.90:0/1216610690 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b457f1700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36cd4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 -- 172.21.15.90:0/1216610690 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 msgr2=0x562f36cd4dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36cd4dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 -- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 msgr2=0x562f36cd5310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd5310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 -- 172.21.15.90:0/1216610690 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b40114b40 con 0x562f36cd5850 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b467f3700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f9b3801ba80 tx=0x7f9b3800e040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=995ba5a3ba433642 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b297fa700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:56.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b38019c70 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b38006e40 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562f36bdf380 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b38019dd0 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b5440ab80 10 client.17820 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.797+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562f36a4cb60 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x562f36bdf380 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b38017440 con 0x562f36cd5850 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 10 client.17820 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 10 client.17820 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 20 client.17820 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36ce8080 0x562f36cea4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 4 client.17820 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:56.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562f36c17700 con 0x562f36ce8080 2024-04-06T14:19:56.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36ce8080 0x562f36cea4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36ce8080 0x562f36cea4a0 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=627b970bb6063916 server_cookie=de61592c18c83c1 in_seq=0 out_seq=0 2024-04-06T14:19:56.803 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.798+0000 7f9b297fa700 10 client.17820 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.799+0000 7f9b297fa700 1 -- 172.21.15.90:0/1216610690 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x562f36c17700 con 0x562f36ce8080 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.799+0000 7f9b297fa700 10 client.17820 handle_command_reply: tid=0 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.799+0000 7f9b5440ab80 1 client.17820 shutdown 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.799+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 msgr2=0x562f36ce3ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.799+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 secure :-1 s=READY pgs=852 cs=0 l=1 rev1=1 crypto rx=0x7f9b3801ba80 tx=0x7f9b3800e040 comp rx=0 tx=0).stop 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 shutdown_connections 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36ce3ab0 unknown :-1 s=CLOSED pgs=852 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36ce8080 0x562f36cea4a0 crc :-1 s=READY pgs=241 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36ccd780 0x562f36cd4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/1216610690 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36cce0a0 0x562f36cd5310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 >> 172.21.15.90:0/1216610690 conn(0x562f36c2c6a0 msgr2=0x562f36cc7c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.800+0000 7f9b297fa700 0 client.17820 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:56.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.803+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 shutdown_connections 2024-04-06T14:19:56.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.803+0000 7f9b5440ab80 1 -- 172.21.15.90:0/1216610690 wait complete. 2024-04-06T14:19:56.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.803+0000 7f9b5440ab80 20 client.17820 trim_cache size 0 max 16384 2024-04-06T14:19:56.820 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.821+0000 7f9b5440ab80 1 Processor -- start 2024-04-06T14:19:56.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.821+0000 7f9b5440ab80 1 -- start start 2024-04-06T14:19:56.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.821+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36ce3860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.821+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cd5850 0x562f36cc88e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.821 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36cce770 con 0x562f36cce0a0 2024-04-06T14:19:56.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36ce4b70 con 0x562f36ccd780 2024-04-06T14:19:56.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36cda7d0 con 0x562f36cd5850 2024-04-06T14:19:56.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60686/0 (socket says 172.21.15.90:60686) 2024-04-06T14:19:56.822 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.822+0000 7f9b457f1700 1 -- 172.21.15.90:0/2688388693 learned_addr learned my addr 172.21.15.90:0/2688388693 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 -- 172.21.15.90:0/2688388693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cd5850 msgr2=0x562f36cc88e0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cd5850 0x562f36cc88e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 -- 172.21.15.90:0/2688388693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 msgr2=0x562f36ce3860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36ce3860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 -- 172.21.15.90:0/2688388693 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f36be4ea0 con 0x562f36cce0a0 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f9b30009ce0 tx=0x7f9b300092d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9ae1f7f667373b2b server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b34ff9700 1 -- 172.21.15.90:0/2688388693 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b3001d040 con 0x562f36cce0a0 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b34ff9700 1 -- 172.21.15.90:0/2688388693 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b3000b2c0 con 0x562f36cce0a0 2024-04-06T14:19:56.823 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.823+0000 7f9b34ff9700 1 -- 172.21.15.90:0/2688388693 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b30004a70 con 0x562f36cce0a0 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.824+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2688388693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 msgr2=0x562f36cc83a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.824+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 secure :-1 s=READY pgs=853 cs=0 l=1 rev1=1 crypto rx=0x7f9b30009ce0 tx=0x7f9b300092d0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2688388693 shutdown_connections 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cd5850 0x562f36cc88e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36ce3860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.824 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2688388693 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cce0a0 0x562f36cc83a0 unknown :-1 s=CLOSED pgs=853 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.825 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2688388693 >> 172.21.15.90:0/2688388693 conn(0x562f36c2c6a0 msgr2=0x562f36ce3150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2688388693 shutdown_connections 2024-04-06T14:19:56.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2688388693 wait complete. 2024-04-06T14:19:56.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 -- start start 2024-04-06T14:19:56.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.825+0000 7f9b5440ab80 1 Processor -- start 2024-04-06T14:19:56.827 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36cde320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36cdc9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36cdd080 con 0x562f36cd5850 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36cdd1f0 con 0x562f36ccd780 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f36cdd3b0 con 0x562f36cce0a0 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b5440ab80 10 client.0 _ll_register_callbacks cb 0x562f36bc4470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55094/0 (socket says 172.21.15.90:55094) 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 -- 172.21.15.90:0/2680254432 learned_addr learned my addr 172.21.15.90:0/2680254432 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b467f3700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36cdc9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36cde320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 -- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 msgr2=0x562f36cde320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.828 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36cde320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 -- 172.21.15.90:0/2680254432 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 msgr2=0x562f36cdc9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36cdc9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b457f1700 1 -- 172.21.15.90:0/2680254432 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562f36aa03b0 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.826+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36cde320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b457f1700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f9b3000b6f0 tx=0x7f9b3000be60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b77a58dba4d90cfd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9b3001d070 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9b30004910 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9b3001e040 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9b300056a0 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 10 client.27286 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562f36aa4cd0 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9b3001e040 con 0x562f36cce0a0 2024-04-06T14:19:56.829 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f9b3000fce0 con 0x562f36cce0a0 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 10 client.27286 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 10 client.27286 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 20 client.27286 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.827+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36d4db80 0x562f36d4ff70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.828+0000 7f9b5440ab80 4 client.27286 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.828+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x562f36a6dcf0 con 0x562f36d4db80 2024-04-06T14:19:56.830 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.828+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36d4db80 0x562f36d4ff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:56.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b45ff2700 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36d4db80 0x562f36d4ff70 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c9ac75d18c27a26a server_cookie=f51c8f7977b0732e in_seq=0 out_seq=0 2024-04-06T14:19:56.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b297fa700 10 client.27286 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:56.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b297fa700 1 -- 172.21.15.90:0/2680254432 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x562f36a6dcf0 con 0x562f36d4db80 2024-04-06T14:19:56.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b297fa700 10 client.27286 handle_command_reply: tid=0 2024-04-06T14:19:56.831 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 client.27286 shutdown 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 msgr2=0x562f36cde860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f9b3000b6f0 tx=0x7f9b3000be60 comp rx=0 tx=0).stop 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 shutdown_connections 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f36cd5850 0x562f36cdc9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f36ccd780 0x562f36cde320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f36d4db80 0x562f36d4ff70 crc :-1 s=READY pgs=242 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 --2- 172.21.15.90:0/2680254432 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f36cce0a0 0x562f36cde860 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b297fa700 0 client.27286 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:56.832 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.829+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 >> 172.21.15.90:0/2680254432 conn(0x562f36c2c6a0 msgr2=0x562f36cd6620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 shutdown_connections 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b5440ab80 1 -- 172.21.15.90:0/2680254432 wait complete. 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b5440ab80 20 client.27286 trim_cache size 0 max 16384 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:19:56.833 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:56.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 msgr2=0x7f9b2c0a3920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 0x7f9b2c0a3920 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f9b3800a4e0 tx=0x7f9b3800c040 comp rx=0 tx=0).stop 2024-04-06T14:19:56.834 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 msgr2=0x7f9b401a3510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 0x7f9b401a3510 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f9b3c00e9f0 tx=0x7f9b3c00ed00 comp rx=0 tx=0).stop 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 shutdown_connections 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9b2c0a1460 0x7f9b2c0a3920 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9b40105c40 0x7f9b4019d640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b400695a0 0x7f9b401a3510 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 --2- 172.21.15.90:0/1712952674 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9b40103740 0x7f9b401a3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 >> 172.21.15.90:0/1712952674 conn(0x7f9b40076d40 msgr2=0x7f9b400fd100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:56.835 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 shutdown_connections 2024-04-06T14:19:56.836 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:56.833+0000 7f9b48256700 1 -- 172.21.15.90:0/1712952674 wait complete. 2024-04-06T14:19:56.848 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:19:56.848 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 -- 172.21.15.90:0/4248538951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 msgr2=0x7f4e64105f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 --2- 172.21.15.90:0/4248538951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64105f60 secure :-1 s=READY pgs=854 cs=0 l=1 rev1=1 crypto rx=0x7f4e58009990 tx=0x7f4e58009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 -- 172.21.15.90:0/4248538951 shutdown_connections 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 --2- 172.21.15.90:0/4248538951 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e641004a0 0x7f4e6410c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 --2- 172.21.15.90:0/4248538951 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64105f60 unknown :-1 s=CLOSED pgs=854 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 --2- 172.21.15.90:0/4248538951 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e640ffae0 0x7f4e640fff60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 -- 172.21.15.90:0/4248538951 >> 172.21.15.90:0/4248538951 conn(0x7f4e640fb5f0 msgr2=0x7f4e640fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 -- 172.21.15.90:0/4248538951 shutdown_connections 2024-04-06T14:19:57.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 -- 172.21.15.90:0/4248538951 wait complete. 2024-04-06T14:19:57.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.042+0000 7f4e6b008700 1 Processor -- start 2024-04-06T14:19:57.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 -- start start 2024-04-06T14:19:57.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e640ffae0 0x7f4e64194e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e641004a0 0x7f4e641953c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e64199da0 con 0x7f4e64105b80 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e64199f10 con 0x7f4e640ffae0 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e6b008700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e6419a0d0 con 0x7f4e641004a0 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60714/0 (socket says 172.21.15.90:60714) 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 -- 172.21.15.90:0/3932464382 learned_addr learned my addr 172.21.15.90:0/3932464382 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e63fff700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e641004a0 0x7f4e641953c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e68da4700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e640ffae0 0x7f4e64194e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 -- 172.21.15.90:0/3932464382 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e641004a0 msgr2=0x7f4e641953c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e641004a0 0x7f4e641953c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 -- 172.21.15.90:0/3932464382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e640ffae0 msgr2=0x7f4e64194e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e640ffae0 0x7f4e64194e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.043+0000 7f4e695a5700 1 -- 172.21.15.90:0/3932464382 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e580095f0 con 0x7f4e64105b80 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e695a5700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f4e5c00ef30 tx=0x7f4e5c00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e5c018070 con 0x7f4e64105b80 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e6419a350 con 0x7f4e64105b80 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4e5c00f040 con 0x7f4e64105b80 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4e5c00cd00 con 0x7f4e64105b80 2024-04-06T14:19:57.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.044+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e641a1bf0 con 0x7f4e64105b80 2024-04-06T14:19:57.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.045+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e641080e0 con 0x7f4e64105b80 2024-04-06T14:19:57.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.047+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4e5c003a00 con 0x7f4e64105b80 2024-04-06T14:19:57.047 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.048+0000 7f4e61ffb700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 0x7f4e540a59e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.048 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.048+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4e5c0cc860 con 0x7f4e64105b80 2024-04-06T14:19:57.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.050+0000 7f4e68da4700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 0x7f4e540a59e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.049 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.050+0000 7f4e68da4700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 0x7f4e540a59e0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f4e5800b2e0 tx=0x7f4e58005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.053 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.054+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4e5c095cc0 con 0x7f4e64105b80 2024-04-06T14:19:57.181 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:57 smithi090 ceph-mon[43420]: pgmap v455: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 16 MiB/s wr, 846 op/s 2024-04-06T14:19:57.430 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.430+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4e641961d0 con 0x7f4e64105b80 2024-04-06T14:19:57.431 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.432+0000 7f4e61ffb700 1 -- 172.21.15.90:0/3932464382 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f4e5c0992e0 con 0x7f4e64105b80 2024-04-06T14:19:57.432 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:19:57.432 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 msgr2=0x7f4e540a59e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 0x7f4e540a59e0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f4e5800b2e0 tx=0x7f4e58005fd0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 msgr2=0x7f4e64199760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 secure :-1 s=READY pgs=855 cs=0 l=1 rev1=1 crypto rx=0x7f4e5c00ef30 tx=0x7f4e5c00c3a0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 shutdown_connections 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4e540a3520 0x7f4e540a59e0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4e64105b80 0x7f4e64199760 unknown :-1 s=CLOSED pgs=855 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4e640ffae0 0x7f4e64194e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 --2- 172.21.15.90:0/3932464382 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4e641004a0 0x7f4e641953c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 >> 172.21.15.90:0/3932464382 conn(0x7f4e640fb5f0 msgr2=0x7f4e640fbe60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 shutdown_connections 2024-04-06T14:19:57.435 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.435+0000 7f4e6b008700 1 -- 172.21.15.90:0/3932464382 wait complete. 2024-04-06T14:19:57.436 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:19:57.451 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:57.451 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:19:57.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:57 smithi105 ceph-mon[46969]: pgmap v455: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 16 MiB/s wr, 846 op/s 2024-04-06T14:19:57.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:57 smithi134 ceph-mon[47549]: pgmap v455: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 59 KiB/s rd, 16 MiB/s wr, 846 op/s 2024-04-06T14:19:57.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.725+0000 7f1600076700 1 -- 172.21.15.90:0/1944175121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f920 msgr2=0x7f15f81082f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.725+0000 7f1600076700 1 --2- 172.21.15.90:0/1944175121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f920 0x7f15f81082f0 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f15f4009990 tx=0x7f15f4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 -- 172.21.15.90:0/1944175121 shutdown_connections 2024-04-06T14:19:57.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 --2- 172.21.15.90:0/1944175121 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f811c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 --2- 172.21.15.90:0/1944175121 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15f810f4b0 0x7f15f8107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 --2- 172.21.15.90:0/1944175121 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f920 0x7f15f81082f0 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 -- 172.21.15.90:0/1944175121 >> 172.21.15.90:0/1944175121 conn(0x7f15f806be00 msgr2=0x7f15f806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 -- 172.21.15.90:0/1944175121 shutdown_connections 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.726+0000 7f1600076700 1 -- 172.21.15.90:0/1944175121 wait complete. 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 Processor -- start 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 -- start start 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 0x7f15f81a6190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15f810f920 0x7f15f81a66d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f81aaa70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15f81ab0b0 con 0x7f15f811a590 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15f81ab220 con 0x7f15f810f4b0 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.727+0000 7f1600076700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f15f81ab3e0 con 0x7f15f810f920 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fde12700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 0x7f15f81a6190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f81aaa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fde12700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 0x7f15f81a6190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53478/0 (socket says 172.21.15.90:53478) 2024-04-06T14:19:57.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fde12700 1 -- 172.21.15.90:0/800351860 learned_addr learned my addr 172.21.15.90:0/800351860 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 -- 172.21.15.90:0/800351860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15f810f920 msgr2=0x7f15f81a66d0 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:19:57.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15f810f920 0x7f15f81a66d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 -- 172.21.15.90:0/800351860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 msgr2=0x7f15f81a6190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 0x7f15f81a6190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 -- 172.21.15.90:0/800351860 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15f40095f0 con 0x7f15f811a590 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15fe613700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f81aaa70 secure :-1 s=READY pgs=856 cs=0 l=1 rev1=1 crypto rx=0x7f15f000bdf0 tx=0x7f15f0009ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f15eeffd700 1 -- 172.21.15.90:0/800351860 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f000cc10 con 0x7f15f811a590 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.728+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15f81ab6c0 con 0x7f15f811a590 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.729+0000 7f15eeffd700 1 -- 172.21.15.90:0/800351860 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15f000cd70 con 0x7f15f811a590 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.729+0000 7f15eeffd700 1 -- 172.21.15.90:0/800351860 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f00126a0 con 0x7f15f811a590 2024-04-06T14:19:57.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.729+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15f81b2f00 con 0x7f15f811a590 2024-04-06T14:19:57.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.732+0000 7f160c22ab80 1 Processor -- start 2024-04-06T14:19:57.732 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f15eeffd700 1 -- 172.21.15.90:0/800351860 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f15f0012860 con 0x7f15f811a590 2024-04-06T14:19:57.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 -- start start 2024-04-06T14:19:57.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff08a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec1ff31d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.733 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1f3e990 con 0x558ec1ff0de0 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1f3c350 con 0x558ec1fe6780 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f160c22ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1f37c40 con 0x558ec1fe70a0 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f15fd611700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f15fd611700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53494/0 (socket says 172.21.15.90:53494) 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f15fd611700 1 -- 172.21.15.90:0/2507824671 learned_addr learned my addr 172.21.15.90:0/2507824671 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.733+0000 7f15fd611700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec1ff31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fde12700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff08a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 -- 172.21.15.90:0/2507824671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 msgr2=0x558ec1ff08a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff08a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 -- 172.21.15.90:0/2507824671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 msgr2=0x558ec1ff31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec1ff31d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.734 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 -- 172.21.15.90:0/2507824671 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558ec1db93b0 con 0x558ec1fe6780 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15eeffd700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 0x7f15e40a3790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fde12700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff08a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15eeffd700 1 -- 172.21.15.90:0/800351860 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f15f0006e40 con 0x7f15f811a590 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15fd611700 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f15f4005b40 tx=0x7f15f4005ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=140299149d16139a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15e3fff700 1 -- 172.21.15.90:0/2507824671 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f4005030 con 0x558ec1fe6780 2024-04-06T14:19:57.735 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15e3fff700 1 -- 172.21.15.90:0/2507824671 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15f4004710 con 0x558ec1fe6780 2024-04-06T14:19:57.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.734+0000 7f15e3fff700 1 -- 172.21.15.90:0/2507824671 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f4017650 con 0x558ec1fe6780 2024-04-06T14:19:57.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.736+0000 7f15fde12700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 0x7f15e40a3790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.736 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 -- 172.21.15.90:0/2507824671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 msgr2=0x558ec1fe6b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f15f4005b40 tx=0x7f15f4005ab0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f15fde12700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 0x7f15e40a3790 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f15e8009c50 tx=0x7f15e8008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 -- 172.21.15.90:0/2507824671 shutdown_connections 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec1ff31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1fe6b60 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2507824671 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff08a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 -- 172.21.15.90:0/2507824671 >> 172.21.15.90:0/2507824671 conn(0x558ec1f456a0 msgr2=0x558ec1fe1a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 -- 172.21.15.90:0/2507824671 shutdown_connections 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.737+0000 7f160c22ab80 1 -- 172.21.15.90:0/2507824671 wait complete. 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 -- start start 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 Processor -- start 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1ff07f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 0x558ec1feda70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1ff0de0 0x558ec1fee0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1ed3d80 con 0x558ec1fe70a0 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1ed3a20 con 0x558ec1fe6780 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1dbb6a0 con 0x558ec1ff0de0 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f160c22ab80 10 client.0 _ll_register_callbacks cb 0x558ec1edd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:57.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f15fd611700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 0x558ec1feda70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f15fd611700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 0x558ec1feda70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60738/0 (socket says 172.21.15.90:60738) 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f15fd611700 1 -- 172.21.15.90:0/2195399502 learned_addr learned my addr 172.21.15.90:0/2195399502 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.738+0000 7f15fde12700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1ff07f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fe613700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1ff0de0 0x558ec1fee0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 -- 172.21.15.90:0/2195399502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1ff0de0 msgr2=0x558ec1fee0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1ff0de0 0x558ec1fee0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 -- 172.21.15.90:0/2195399502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 msgr2=0x558ec1feda70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 0x558ec1feda70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 -- 172.21.15.90:0/2195399502 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15f811d270 con 0x558ec1fe6780 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15fde12700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1ff07f0 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f15e80118d0 tx=0x7f15e8004750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=983757dbd1f76146 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15e17fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:57.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15e8009d90 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f15e8021440 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558ec1ef8380 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f15e8021640 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f160c22ab80 10 client.27575 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.739+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x558ec1d65b60 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.740+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x558ec1ef8380 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.740+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f15e8014730 con 0x558ec1fe6780 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.740+0000 7f160c22ab80 10 client.27575 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.740+0000 7f160c22ab80 10 client.27575 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.741+0000 7f160c22ab80 20 client.27575 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.741+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec2000920 0x558ec2002d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.741+0000 7f160c22ab80 4 client.27575 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:57.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.741+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558ec1f30700 con 0x558ec2000920 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.741+0000 7f15fd611700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec2000920 0x558ec2002d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f15fd611700 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec2000920 0x558ec2002d40 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=7d542ce24d4ea9c6 server_cookie=91895753d9659b1c in_seq=0 out_seq=0 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f15e17fa700 10 client.27575 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f15e17fa700 1 -- 172.21.15.90:0/2195399502 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x558ec1f30700 con 0x558ec2000920 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f15e17fa700 10 client.27575 handle_command_reply: tid=0 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 client.27575 shutdown 2024-04-06T14:19:57.744 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 msgr2=0x558ec1ff07f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1ff07f0 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f15e80118d0 tx=0x7f15e8004750 comp rx=0 tx=0).stop 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 shutdown_connections 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec2000920 0x558ec2002d40 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1ff0de0 0x558ec1fee0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe6780 0x558ec1ff07f0 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 --2- 172.21.15.90:0/2195399502 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe70a0 0x558ec1feda70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 >> 172.21.15.90:0/2195399502 conn(0x558ec1f456a0 msgr2=0x558ec1fe0c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.742+0000 7f15e17fa700 0 client.27575 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:57.745 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.743+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 shutdown_connections 2024-04-06T14:19:57.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.743+0000 7f160c22ab80 1 -- 172.21.15.90:0/2195399502 wait complete. 2024-04-06T14:19:57.746 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.743+0000 7f160c22ab80 20 client.27575 trim_cache size 0 max 16384 2024-04-06T14:19:57.760 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.760+0000 7f160c22ab80 1 Processor -- start 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.761+0000 7f160c22ab80 1 -- start start 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe6780 0x558ec1ff31d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 0x558ec1ff0320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1fe2dc0 con 0x558ec1fe6780 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1ff7f40 con 0x558ec1ff0de0 2024-04-06T14:19:57.761 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f160c22ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec1fe8880 con 0x558ec1fe70a0 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fd611700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fd611700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55152/0 (socket says 172.21.15.90:55152) 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fd611700 1 -- 172.21.15.90:0/4224575827 learned_addr learned my addr 172.21.15.90:0/4224575827 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 0x558ec1ff0320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 -- 172.21.15.90:0/4224575827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 msgr2=0x558ec1ff7a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 -- 172.21.15.90:0/4224575827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe6780 msgr2=0x558ec1ff31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.763 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe6780 0x558ec1ff31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 -- 172.21.15.90:0/4224575827 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558ec1efdea0 con 0x558ec1ff0de0 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fd611700 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.762+0000 7f15fe613700 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 0x558ec1ff0320 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f15f00c9d60 tx=0x7f15f0004be0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d28391cdada7b653 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f15ecff9700 1 -- 172.21.15.90:0/4224575827 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f009da30 con 0x558ec1ff0de0 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f15ecff9700 1 -- 172.21.15.90:0/4224575827 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15f00a2080 con 0x558ec1ff0de0 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f15ecff9700 1 -- 172.21.15.90:0/4224575827 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f00850d0 con 0x558ec1ff0de0 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 -- 172.21.15.90:0/4224575827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 msgr2=0x558ec1ff0320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 0x558ec1ff0320 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f15f00c9d60 tx=0x7f15f0004be0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 -- 172.21.15.90:0/4224575827 shutdown_connections 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1ff0de0 0x558ec1ff0320 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1fe6780 0x558ec1ff31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 --2- 172.21.15.90:0/4224575827 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe70a0 0x558ec1ff7a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.764 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 -- 172.21.15.90:0/4224575827 >> 172.21.15.90:0/4224575827 conn(0x558ec1f456a0 msgr2=0x558ec1fed300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 -- 172.21.15.90:0/4224575827 shutdown_connections 2024-04-06T14:19:57.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.763+0000 7f160c22ab80 1 -- 172.21.15.90:0/4224575827 wait complete. 2024-04-06T14:19:57.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 -- start start 2024-04-06T14:19:57.765 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 Processor -- start 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe6780 0x558ec1ff5710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec2053750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec2053c90 con 0x558ec1ff0de0 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec2053dd0 con 0x558ec1fe70a0 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ec2053f10 con 0x558ec1fe6780 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.764+0000 7f160c22ab80 10 client.0 _ll_register_callbacks cb 0x558ec1edd470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:57.768 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53520/0 (socket says 172.21.15.90:53520) 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 -- 172.21.15.90:0/3405963629 learned_addr learned my addr 172.21.15.90:0/3405963629 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fde12700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe6780 0x558ec1ff5710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fe613700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec2053750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 -- 172.21.15.90:0/3405963629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe6780 msgr2=0x558ec1ff5710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe6780 0x558ec1ff5710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 -- 172.21.15.90:0/3405963629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 msgr2=0x558ec2053750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec2053750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 -- 172.21.15.90:0/3405963629 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558ec1db93b0 con 0x558ec1fe70a0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fe613700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec2053750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15fd611700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f15f400b580 tx=0x7f15f4004f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=be867db665378ae0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15e17fa700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:19:57.769 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f15f402e040 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.765+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f15f4004710 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f15f4035390 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f15f400b980 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f160c22ab80 10 client.27581 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f15f81b2f00 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f15f4004710 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.766+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f15f400b260 con 0x558ec1fe70a0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 10 client.27581 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 10 client.27581 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 20 client.27581 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec20552f0 0x558ec2057710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 4 client.27581 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x558ec1d86cf0 con 0x558ec20552f0 2024-04-06T14:19:57.770 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.767+0000 7f15fde12700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec20552f0 0x558ec2057710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f15e17fa700 10 client.27581 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f15fde12700 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec20552f0 0x558ec2057710 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=aea23b93ff64b51a server_cookie=d19e2b89880359c0 in_seq=0 out_seq=0 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f15e17fa700 1 -- 172.21.15.90:0/3405963629 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x558ec1d86cf0 con 0x558ec20552f0 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f15e17fa700 10 client.27581 handle_command_reply: tid=0 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 client.27581 shutdown 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 msgr2=0x558ec1ff5c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f15f400b580 tx=0x7f15f4004f40 comp rx=0 tx=0).stop 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 shutdown_connections 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x558ec20552f0 0x558ec2057710 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x558ec1ff0de0 0x558ec2053750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f15e17fa700 0 client.27581 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x558ec1fe6780 0x558ec1ff5710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 --2- 172.21.15.90:0/3405963629 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558ec1fe70a0 0x558ec1ff5c50 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.771 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 >> 172.21.15.90:0/3405963629 conn(0x558ec1f456a0 msgr2=0x558ec1fec1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 shutdown_connections 2024-04-06T14:19:57.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.768+0000 7f160c22ab80 1 -- 172.21.15.90:0/3405963629 wait complete. 2024-04-06T14:19:57.772 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f160c22ab80 20 client.27581 trim_cache size 0 max 16384 2024-04-06T14:19:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 msgr2=0x7f15e40a3790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 0x7f15e40a3790 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f15e8009c50 tx=0x7f15e8008040 comp rx=0 tx=0).stop 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 msgr2=0x7f15f81aaa70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f81aaa70 secure :-1 s=READY pgs=856 cs=0 l=1 rev1=1 crypto rx=0x7f15f000bdf0 tx=0x7f15f0009ed0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 shutdown_connections 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f15e40a12d0 0x7f15e40a3790 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f15f811a590 0x7f15f81aaa70 unknown :-1 s=CLOSED pgs=856 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f15f810f4b0 0x7f15f81a6190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.773 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 --2- 172.21.15.90:0/800351860 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f15f810f920 0x7f15f81a66d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 >> 172.21.15.90:0/800351860 conn(0x7f15f806be00 msgr2=0x7f15f8118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 shutdown_connections 2024-04-06T14:19:57.774 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.769+0000 7f1600076700 1 -- 172.21.15.90:0/800351860 wait complete. 2024-04-06T14:19:57.786 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:57.787 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 -- 172.21.15.90:0/2541160047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23811a590 msgr2=0x7fb23811c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 --2- 172.21.15.90:0/2541160047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23811a590 0x7fb23811c980 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7fb23001c0d0 tx=0x7fb23001c3e0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 -- 172.21.15.90:0/2541160047 shutdown_connections 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 --2- 172.21.15.90:0/2541160047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23811a590 0x7fb23811c980 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 --2- 172.21.15.90:0/2541160047 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb23810f4b0 0x7fb238107db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.975 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 --2- 172.21.15.90:0/2541160047 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb23810f920 0x7fb2381082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.976 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 -- 172.21.15.90:0/2541160047 >> 172.21.15.90:0/2541160047 conn(0x7fb23806be00 msgr2=0x7fb23806c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 -- 172.21.15.90:0/2541160047 shutdown_connections 2024-04-06T14:19:57.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.975+0000 7fb23f292700 1 -- 172.21.15.90:0/2541160047 wait complete. 2024-04-06T14:19:57.977 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.976+0000 7fb23f292700 1 Processor -- start 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.976+0000 7fb23f292700 1 -- start start 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.976+0000 7fb23f292700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb23810f4b0 0x7fb238119c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23f292700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23f292700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb23811a590 0x7fb238116210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23f292700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb238116810 con 0x7fb23810f4b0 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23f292700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb238116980 con 0x7fb23810f920 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23f292700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb238116b40 con 0x7fb23811a590 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53540/0 (socket says 172.21.15.90:53540) 2024-04-06T14:19:57.978 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 -- 172.21.15.90:0/1135909002 learned_addr learned my addr 172.21.15.90:0/1135909002 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 -- 172.21.15.90:0/1135909002 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb23811a590 msgr2=0x7fb238116210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb23811a590 0x7fb238116210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 -- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb23810f4b0 msgr2=0x7fb238119c60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb23810f4b0 0x7fb238119c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 -- 172.21.15.90:0/1135909002 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb23001ad10 con 0x7fb23810f920 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.977+0000 7fb23c82d700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fb22c00b530 tx=0x7fb22c00b8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.978+0000 7fb22a7fc700 1 -- 172.21.15.90:0/1135909002 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb22c011830 con 0x7fb23810f920 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.978+0000 7fb22a7fc700 1 -- 172.21.15.90:0/1135909002 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb22c011e50 con 0x7fb23810f920 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.978+0000 7fb22a7fc700 1 -- 172.21.15.90:0/1135909002 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb22c00f2e0 con 0x7fb23810f920 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.978+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2381aa2e0 con 0x7fb23810f920 2024-04-06T14:19:57.979 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.978+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2381aa7a0 con 0x7fb23810f920 2024-04-06T14:19:57.980 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.981+0000 7fb22a7fc700 1 -- 172.21.15.90:0/1135909002 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb22c0105a0 con 0x7fb23810f920 2024-04-06T14:19:57.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.982+0000 7fb22a7fc700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 0x7fb2240a38a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.981 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.982+0000 7fb22a7fc700 1 -- 172.21.15.90:0/1135909002 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb22c0cc600 con 0x7fb23810f920 2024-04-06T14:19:57.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 Processor -- start 2024-04-06T14:19:57.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb23d02e700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 0x7fb2240a38a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.982 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 -- start start 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbedd3b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd40a0 0x55dcbeddd8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbed2b990 con 0x55dcbedddde0 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbed29350 con 0x55dcbedd3780 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.983+0000 7fb24b446b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbed24c40 con 0x55dcbedd40a0 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d02e700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 0x7fb2240a38a0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fb2340097b0 tx=0x7fb234006d20 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.985 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60776/0 (socket says 172.21.15.90:60776) 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 -- 172.21.15.90:0/1746392941 learned_addr learned my addr 172.21.15.90:0/1746392941 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d02e700 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbedd3b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23c82d700 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd40a0 0x55dcbeddd8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 -- 172.21.15.90:0/1746392941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd40a0 msgr2=0x55dcbeddd8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd40a0 0x55dcbeddd8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 -- 172.21.15.90:0/1746392941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 msgr2=0x55dcbedd3b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbedd3b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.986 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 -- 172.21.15.90:0/1746392941 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dcbeba63b0 con 0x55dcbedddde0 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb23d82f700 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7fb23001a180 tx=0x7fb23001a1b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a4a51beb95064aa6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb2237fe700 1 -- 172.21.15.90:0/1746392941 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb23001f040 con 0x55dcbedddde0 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb2237fe700 1 -- 172.21.15.90:0/1746392941 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb230018b90 con 0x55dcbedddde0 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.984+0000 7fb2237fe700 1 -- 172.21.15.90:0/1746392941 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2300052d0 con 0x55dcbedddde0 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 -- 172.21.15.90:0/1746392941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 msgr2=0x55dcbede01d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 secure :-1 s=READY pgs=857 cs=0 l=1 rev1=1 crypto rx=0x7fb23001a180 tx=0x7fb23001a1b0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 -- 172.21.15.90:0/1746392941 shutdown_connections 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedddde0 0x55dcbede01d0 unknown :-1 s=CLOSED pgs=857 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbedd3b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 --2- 172.21.15.90:0/1746392941 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd40a0 0x55dcbeddd8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 -- 172.21.15.90:0/1746392941 >> 172.21.15.90:0/1746392941 conn(0x55dcbed326a0 msgr2=0x55dcbedcea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.987 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 -- 172.21.15.90:0/1746392941 shutdown_connections 2024-04-06T14:19:57.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.985+0000 7fb24b446b80 1 -- 172.21.15.90:0/1746392941 wait complete. 2024-04-06T14:19:57.988 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 -- start start 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 Processor -- start 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd3780 0x55dcbeddd760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedddde0 0x55dcbeddb120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbecc0d80 con 0x55dcbedd40a0 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbecc0a20 con 0x55dcbedddde0 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbeba86a0 con 0x55dcbedd3780 2024-04-06T14:19:57.989 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.986+0000 7fb24b446b80 10 client.0 _ll_register_callbacks cb 0x55dcbecca470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.987+0000 7fb23c82d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.987+0000 7fb23c82d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60778/0 (socket says 172.21.15.90:60778) 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.987+0000 7fb23c82d700 1 -- 172.21.15.90:0/708250257 learned_addr learned my addr 172.21.15.90:0/708250257 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23d02e700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd3780 0x55dcbeddd760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 -- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd3780 msgr2=0x55dcbeddd760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd3780 0x55dcbeddd760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 -- 172.21.15.90:0/708250257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedddde0 msgr2=0x55dcbeddb120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedddde0 0x55dcbeddb120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 -- 172.21.15.90:0/708250257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb23811d270 con 0x55dcbedd40a0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb23c82d700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7fb22c00bcf0 tx=0x7fb22c09a050 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2290487652c9fadc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.988+0000 7fb220ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:57.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.989+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb22c0b90b0 con 0x55dcbedd40a0 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.989+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb22c0b9210 con 0x55dcbedd40a0 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.989+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55dcbece5380 con 0x55dcbedd40a0 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb22c099a60 con 0x55dcbedd40a0 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 10 client.17850 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dcbeb52b60 con 0x55dcbedd40a0 2024-04-06T14:19:57.991 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55dcbece5380 con 0x55dcbedd40a0 2024-04-06T14:19:57.992 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb22c0a17f0 con 0x55dcbedd40a0 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 10 client.17850 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 10 client.17850 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 20 client.17850 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbeded930 0x55dcbedefd50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 4 client.17850 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55dcbed1d700 con 0x55dcbeded930 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.990+0000 7fb23d02e700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbeded930 0x55dcbedefd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.991+0000 7fb220ff9700 10 client.17850 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.991+0000 7fb23d02e700 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbeded930 0x55dcbedefd50 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=211fbdac513c7cfe server_cookie=3b4c3fc628adc301 in_seq=0 out_seq=0 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.991+0000 7fb220ff9700 1 -- 172.21.15.90:0/708250257 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55dcbed1d700 con 0x55dcbeded930 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.991+0000 7fb220ff9700 10 client.17850 handle_command_reply: tid=0 2024-04-06T14:19:57.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 client.17850 shutdown 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 msgr2=0x55dcbeddaaf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 secure :-1 s=READY pgs=858 cs=0 l=1 rev1=1 crypto rx=0x7fb22c00bcf0 tx=0x7fb22c09a050 comp rx=0 tx=0).stop 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 shutdown_connections 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbeded930 0x55dcbedefd50 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedddde0 0x55dcbeddb120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedd3780 0x55dcbeddd760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 --2- 172.21.15.90:0/708250257 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbeddaaf0 unknown :-1 s=CLOSED pgs=858 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 >> 172.21.15.90:0/708250257 conn(0x55dcbed326a0 msgr2=0x55dcbedcdc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.992+0000 7fb220ff9700 0 client.17850 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.993+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 shutdown_connections 2024-04-06T14:19:57.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.993+0000 7fb24b446b80 1 -- 172.21.15.90:0/708250257 wait complete. 2024-04-06T14:19:57.995 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:57.993+0000 7fb24b446b80 20 client.17850 trim_cache size 0 max 16384 2024-04-06T14:19:58.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 Processor -- start 2024-04-06T14:19:58.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 -- start start 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 0x55dcbede4a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 0x55dcbede9860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbedce3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbedd4a10 con 0x55dcbedd3780 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbeddba00 con 0x55dcbedd40a0 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb24b446b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbede4f40 con 0x55dcbedddde0 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbedce3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbedce3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55194/0 (socket says 172.21.15.90:55194) 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.010+0000 7fb23d82f700 1 -- 172.21.15.90:0/121817258 learned_addr learned my addr 172.21.15.90:0/121817258 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 0x55dcbede9860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.011 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23d02e700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 0x55dcbede4a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 -- 172.21.15.90:0/121817258 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 msgr2=0x55dcbedce3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbedce3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 -- 172.21.15.90:0/121817258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 msgr2=0x55dcbede4a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 0x55dcbede4a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 -- 172.21.15.90:0/121817258 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55dcbeceaea0 con 0x55dcbedd40a0 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23d02e700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 0x55dcbede4a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.011+0000 7fb23c82d700 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 0x55dcbede9860 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fb22c099500 tx=0x7fb22c097db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c19dde9040895cb7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.012 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.013+0000 7fb223fff700 1 -- 172.21.15.90:0/121817258 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb22c0995d0 con 0x55dcbedd40a0 2024-04-06T14:19:58.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.013+0000 7fb223fff700 1 -- 172.21.15.90:0/121817258 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb22c0a17f0 con 0x55dcbedd40a0 2024-04-06T14:19:58.013 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.013+0000 7fb223fff700 1 -- 172.21.15.90:0/121817258 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb22c099a60 con 0x55dcbedd40a0 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 -- 172.21.15.90:0/121817258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 msgr2=0x55dcbede9860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 0x55dcbede9860 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 crypto rx=0x7fb22c099500 tx=0x7fb22c097db0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 -- 172.21.15.90:0/121817258 shutdown_connections 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbedce3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd3780 0x55dcbede4a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 --2- 172.21.15.90:0/121817258 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd40a0 0x55dcbede9860 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.014 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.014+0000 7fb24b446b80 1 -- 172.21.15.90:0/121817258 >> 172.21.15.90:0/121817258 conn(0x55dcbed326a0 msgr2=0x55dcbede9150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.015 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.016+0000 7fb24b446b80 1 -- 172.21.15.90:0/121817258 shutdown_connections 2024-04-06T14:19:58.016 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.017+0000 7fb24b446b80 1 -- 172.21.15.90:0/121817258 wait complete. 2024-04-06T14:19:58.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.017+0000 7fb24b446b80 1 -- start start 2024-04-06T14:19:58.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.017+0000 7fb24b446b80 1 Processor -- start 2024-04-06T14:19:58.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbede3f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.017 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbede4470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbee3ace0 con 0x55dcbedd40a0 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbee3ae50 con 0x55dcbedd3780 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55dcbee3b010 con 0x55dcbedddde0 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb24b446b80 10 client.0 _ll_register_callbacks cb 0x55dcbecca470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55208/0 (socket says 172.21.15.90:55208) 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 -- 172.21.15.90:0/4131656047 learned_addr learned my addr 172.21.15.90:0/4131656047 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.018 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23c82d700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbede4470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d02e700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbede3f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 -- 172.21.15.90:0/4131656047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 msgr2=0x55dcbede3f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbede3f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 -- 172.21.15.90:0/4131656047 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 msgr2=0x55dcbede4470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbede4470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.018+0000 7fb23d82f700 1 -- 172.21.15.90:0/4131656047 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55dcbeb52b60 con 0x55dcbedddde0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb23d82f700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fb23001a860 tx=0x7fb230003a00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1085853199c12768 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb220ff9700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb23001f050 con 0x55dcbedddde0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fb230018b90 con 0x55dcbedddde0 2024-04-06T14:19:58.021 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb2300186f0 con 0x55dcbedddde0 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb230003e60 con 0x55dcbedddde0 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb24b446b80 10 client.27316 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55dcbebaacd0 con 0x55dcbedddde0 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.019+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb230018b90 con 0x55dcbedddde0 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fb23001c500 con 0x55dcbedddde0 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 10 client.27316 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 10 client.27316 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 20 client.27316 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbee42510 0x55dcbee44900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 4 client.27316 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:58.022 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55dcbeb73cf0 con 0x55dcbee42510 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.020+0000 7fb23d02e700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbee42510 0x55dcbee44900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb220ff9700 10 client.27316 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb23d02e700 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbee42510 0x55dcbee44900 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=370a5a01350d84b1 server_cookie=d2e6e1011be3f61c in_seq=0 out_seq=0 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb220ff9700 1 -- 172.21.15.90:0/4131656047 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55dcbeb73cf0 con 0x55dcbee42510 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb220ff9700 10 client.27316 handle_command_reply: tid=0 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 client.27316 shutdown 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 msgr2=0x55dcbee3a730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fb23001a860 tx=0x7fb230003a00 comp rx=0 tx=0).stop 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 shutdown_connections 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55dcbee42510 0x55dcbee44900 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55dcbedddde0 0x55dcbee3a730 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55dcbedd3780 0x55dcbede3f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 --2- 172.21.15.90:0/4131656047 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55dcbedd40a0 0x55dcbede4470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.023 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 >> 172.21.15.90:0/4131656047 conn(0x55dcbed326a0 msgr2=0x55dcbede99e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb220ff9700 0 client.27316 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:19:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 shutdown_connections 2024-04-06T14:19:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 1 -- 172.21.15.90:0/4131656047 wait complete. 2024-04-06T14:19:58.024 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.021+0000 7fb24b446b80 20 client.27316 trim_cache size 0 max 16384 2024-04-06T14:19:58.024 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 msgr2=0x7fb2240a38a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 0x7fb2240a38a0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7fb2340097b0 tx=0x7fb234006d20 comp rx=0 tx=0).stop 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 msgr2=0x7fb238115cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fb22c00b530 tx=0x7fb22c00b8f0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 shutdown_connections 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb2240a13e0 0x7fb2240a38a0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb23811a590 0x7fb238116210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.025 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb23810f4b0 0x7fb238119c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 --2- 172.21.15.90:0/1135909002 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb23810f920 0x7fb238115cd0 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 >> 172.21.15.90:0/1135909002 conn(0x7fb23806be00 msgr2=0x7fb23811a970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 shutdown_connections 2024-04-06T14:19:58.026 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.022+0000 7fb23f292700 1 -- 172.21.15.90:0/1135909002 wait complete. 2024-04-06T14:19:58.026 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 850.0 s, 1432.8 tps, lat 69.424 ms stddev 80.096 2024-04-06T14:19:58.036 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:19:58.036 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:19:58.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 -- 172.21.15.90:0/2998327545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b40104f70 msgr2=0x7f5b4010caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 --2- 172.21.15.90:0/2998327545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b40104f70 0x7f5b4010caf0 secure :-1 s=READY pgs=859 cs=0 l=1 rev1=1 crypto rx=0x7f5b3800b0a0 tx=0x7f5b3800b3b0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 -- 172.21.15.90:0/2998327545 shutdown_connections 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 --2- 172.21.15.90:0/2998327545 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b40104f70 0x7f5b4010caf0 unknown :-1 s=CLOSED pgs=859 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 --2- 172.21.15.90:0/2998327545 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b400ffe70 0x7f5b40100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 --2- 172.21.15.90:0/2998327545 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b40100820 0x7f5b40104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 -- 172.21.15.90:0/2998327545 >> 172.21.15.90:0/2998327545 conn(0x7f5b40076d50 msgr2=0x7f5b40077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 -- 172.21.15.90:0/2998327545 shutdown_connections 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 -- 172.21.15.90:0/2998327545 wait complete. 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.221+0000 7f5b460e9700 1 Processor -- start 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 -- start start 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 0x7f5b4019df80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 0x7f5b4019e4e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b40109b60 0x7f5b40109ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b4010a7c0 con 0x7f5b400ffe70 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b4010a930 con 0x7f5b40104f70 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b460e9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b40199a30 con 0x7f5b40109b60 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 0x7f5b4019df80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 0x7f5b4019df80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60820/0 (socket says 172.21.15.90:60820) 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/3047605771 learned_addr learned my addr 172.21.15.90:0/3047605771 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.222+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 0x7f5b4019e4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 -- 172.21.15.90:0/3047605771 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b40109b60 msgr2=0x7f5b40109ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b40109b60 0x7f5b40109ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 -- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 msgr2=0x7f5b4019df80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 0x7f5b4019df80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 -- 172.21.15.90:0/3047605771 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b38009d00 con 0x7f5b40104f70 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 0x7f5b4019e4e0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f5b3400e9c0 tx=0x7f5b3400ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3cff9700 1 -- 172.21.15.90:0/3047605771 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b3400cbd0 con 0x7f5b40104f70 2024-04-06T14:19:58.224 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3cff9700 1 -- 172.21.15.90:0/3047605771 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b3400eeb0 con 0x7f5b40104f70 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b40199ce0 con 0x7f5b40104f70 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b3cff9700 1 -- 172.21.15.90:0/3047605771 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b34018680 con 0x7f5b40104f70 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.223+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b4019a1f0 con 0x7f5b40104f70 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.225+0000 7f5b3cff9700 1 -- 172.21.15.90:0/3047605771 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5b340187e0 con 0x7f5b40104f70 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.226+0000 7f5b3cff9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 0x7f5b280a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.225 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.226+0000 7f5b3cff9700 1 -- 172.21.15.90:0/3047605771 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5b34014070 con 0x7f5b40104f70 2024-04-06T14:19:58.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.226+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 0x7f5b280a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 Processor -- start 2024-04-06T14:19:58.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 -- start start 2024-04-06T14:19:58.228 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eef5b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef60a0 0x562f0eeff8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ee4d990 con 0x562f0eef60a0 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.227+0000 7f5b5229db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ee4b350 con 0x562f0eeffde0 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b5229db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ee46c40 con 0x562f0eef5780 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 0x7f5b280a3750 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f5b30009d20 tx=0x7f5b30009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eef5b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:53592/0 (socket says 172.21.15.90:53592) 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 -- 172.21.15.90:0/2376768375 learned_addr learned my addr 172.21.15.90:0/2376768375 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 -- 172.21.15.90:0/2376768375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 msgr2=0x562f0eef5b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eef5b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.229 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 -- 172.21.15.90:0/2376768375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef60a0 msgr2=0x562f0eeff8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef60a0 0x562f0eeff8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 -- 172.21.15.90:0/2376768375 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f0ecc83b0 con 0x562f0eeffde0 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.228+0000 7f5b3ffff700 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f5b38006070 tx=0x7f5b380092e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f69881a1d26f2de9 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.229+0000 7f5b25ffb700 1 -- 172.21.15.90:0/2376768375 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b3800e040 con 0x562f0eeffde0 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.229+0000 7f5b25ffb700 1 -- 172.21.15.90:0/2376768375 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b38009490 con 0x562f0eeffde0 2024-04-06T14:19:58.230 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.229+0000 7f5b25ffb700 1 -- 172.21.15.90:0/2376768375 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b38003a00 con 0x562f0eeffde0 2024-04-06T14:19:58.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 -- 172.21.15.90:0/2376768375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 msgr2=0x562f0ef021d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f5b38006070 tx=0x7f5b380092e0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.232 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 -- 172.21.15.90:0/2376768375 shutdown_connections 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef021d0 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eef5b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2376768375 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef60a0 0x562f0eeff8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.230+0000 7f5b5229db80 1 -- 172.21.15.90:0/2376768375 >> 172.21.15.90:0/2376768375 conn(0x562f0ee546a0 msgr2=0x562f0eef0a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.231+0000 7f5b5229db80 1 -- 172.21.15.90:0/2376768375 shutdown_connections 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.231+0000 7f5b5229db80 1 -- 172.21.15.90:0/2376768375 wait complete. 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.231+0000 7f5b5229db80 1 -- start start 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.231+0000 7f5b5229db80 1 Processor -- start 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eefcb90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eeffde0 0x562f0eefd2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ede2d80 con 0x562f0eeffde0 2024-04-06T14:19:58.233 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ede2a20 con 0x562f0eef60a0 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ecca6a0 con 0x562f0eef5780 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b5229db80 10 client.0 _ll_register_callbacks cb 0x562f0edec470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55246/0 (socket says 172.21.15.90:55246) 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/3094010103 learned_addr learned my addr 172.21.15.90:0/3094010103 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eefcb90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3ffff700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eeffde0 0x562f0eefd2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 msgr2=0x562f0eefcb90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eefcb90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/3094010103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eeffde0 msgr2=0x562f0eefd2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eeffde0 0x562f0eefd2d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.232+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/3094010103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b40067e00 con 0x562f0eef5780 2024-04-06T14:19:58.234 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f5b3001dc00 tx=0x7f5b300047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=712fdc18b577e4ed server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b1effd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b30022070 con 0x562f0eef5780 2024-04-06T14:19:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5b3000e9d0 con 0x562f0eef5780 2024-04-06T14:19:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562f0ee07380 con 0x562f0eef5780 2024-04-06T14:19:58.235 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.233+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5b3000eb90 con 0x562f0eef5780 2024-04-06T14:19:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 10 client.27325 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:58.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562f0ec74b60 con 0x562f0eef5780 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x562f0ee07380 con 0x562f0eef5780 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5b30027420 con 0x562f0eef5780 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 10 client.27325 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 10 client.27325 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 20 client.27325 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef0f9e0 0x562f0ef11e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 4 client.27325 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562f0ee3f700 con 0x562f0ef0f9e0 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.234+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef0f9e0 0x562f0ef11e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.235+0000 7f5b1effd700 10 client.27325 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.235+0000 7f5b3effd700 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef0f9e0 0x562f0ef11e00 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6fcdd4a825532b7d server_cookie=3ab4808755f33d2f in_seq=0 out_seq=0 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.235+0000 7f5b1effd700 1 -- 172.21.15.90:0/3094010103 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x562f0ee3f700 con 0x562f0ef0f9e0 2024-04-06T14:19:58.237 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.235+0000 7f5b1effd700 10 client.27325 handle_command_reply: tid=0 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.235+0000 7f5b5229db80 1 client.27325 shutdown 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 msgr2=0x562f0eeff8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f5b3001dc00 tx=0x7f5b300047c0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 shutdown_connections 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eeffde0 0x562f0eefd2d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef0f9e0 0x562f0ef11e00 crc :-1 s=READY pgs=243 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef5780 0x562f0eeff8a0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b1effd700 0 client.27325 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 --2- 172.21.15.90:0/3094010103 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eefcb90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 >> 172.21.15.90:0/3094010103 conn(0x562f0ee546a0 msgr2=0x562f0eeefc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 shutdown_connections 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 1 -- 172.21.15.90:0/3094010103 wait complete. 2024-04-06T14:19:58.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.236+0000 7f5b5229db80 20 client.27325 trim_cache size 0 max 16384 2024-04-06T14:19:58.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.255+0000 7f5b5229db80 1 Processor -- start 2024-04-06T14:19:58.255 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 -- start start 2024-04-06T14:19:58.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eeff3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eeffde0 0x562f0eef03a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0eefdb00 con 0x562f0eef5780 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0eefb050 con 0x562f0eef60a0 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b5229db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ef021d0 con 0x562f0eeffde0 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b3ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eeffde0 0x562f0eef03a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b3f7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60856/0 (socket says 172.21.15.90:60856) 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/2568282472 learned_addr learned my addr 172.21.15.90:0/2568282472 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.256+0000 7f5b3effd700 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eeff3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/2568282472 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eeffde0 msgr2=0x562f0eef03a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eeffde0 0x562f0eef03a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/2568282472 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 msgr2=0x562f0eeff3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eeff3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 -- 172.21.15.90:0/2568282472 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f0ee0cea0 con 0x562f0eef5780 2024-04-06T14:19:58.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f5b30000c00 tx=0x7f5b3000e690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f370620ee0b6cae server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b267fc700 1 -- 172.21.15.90:0/2568282472 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b30022070 con 0x562f0eef5780 2024-04-06T14:19:58.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b267fc700 1 -- 172.21.15.90:0/2568282472 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b3000ee90 con 0x562f0eef5780 2024-04-06T14:19:58.258 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.257+0000 7f5b267fc700 1 -- 172.21.15.90:0/2568282472 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b300315f0 con 0x562f0eef5780 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.258+0000 7f5b5229db80 1 -- 172.21.15.90:0/2568282472 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 msgr2=0x562f0eef7630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.258+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 secure :-1 s=READY pgs=860 cs=0 l=1 rev1=1 crypto rx=0x7f5b30000c00 tx=0x7f5b3000e690 comp rx=0 tx=0).stop 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 -- 172.21.15.90:0/2568282472 shutdown_connections 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eeffde0 0x562f0eef03a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0eef7630 unknown :-1 s=CLOSED pgs=860 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 --2- 172.21.15.90:0/2568282472 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eef60a0 0x562f0eeff3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 -- 172.21.15.90:0/2568282472 >> 172.21.15.90:0/2568282472 conn(0x562f0ee546a0 msgr2=0x562f0ef16720 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 -- 172.21.15.90:0/2568282472 shutdown_connections 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 -- 172.21.15.90:0/2568282472 wait complete. 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 -- start start 2024-04-06T14:19:58.259 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.259+0000 7f5b5229db80 1 Processor -- start 2024-04-06T14:19:58.260 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0ef05120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef05ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ef637e0 con 0x562f0eef5780 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ef63920 con 0x562f0eeffde0 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0ef63a60 con 0x562f0eef60a0 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b5229db80 10 client.0 _ll_register_callbacks cb 0x562f0edec470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b3effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b3effd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55258/0 (socket says 172.21.15.90:55258) 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.260+0000 7f5b3effd700 1 -- 172.21.15.90:0/975342298 learned_addr learned my addr 172.21.15.90:0/975342298 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3ffff700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef05ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 -- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 msgr2=0x562f0ef05ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef05ba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 -- 172.21.15.90:0/975342298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 msgr2=0x562f0ef05120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:58.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0ef05120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 -- 172.21.15.90:0/975342298 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562f0ec74b60 con 0x562f0eef60a0 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b3effd700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f5b34010b20 tx=0x7f5b34015000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=555c8a19076c1462 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b1effd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b3409ed40 con 0x562f0eef60a0 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5b3409a400 con 0x562f0eef60a0 2024-04-06T14:19:58.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.261+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5b38009490 con 0x562f0eef60a0 2024-04-06T14:19:58.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.262+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5b3409a5c0 con 0x562f0eef60a0 2024-04-06T14:19:58.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.263+0000 7f5b5229db80 10 client.27331 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:58.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.263+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x562f0eccccd0 con 0x562f0eef60a0 2024-04-06T14:19:58.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.264+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b34064070 con 0x562f0eef60a0 2024-04-06T14:19:58.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f5b3409b170 con 0x562f0eef60a0 2024-04-06T14:19:58.266 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 10 client.27331 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 10 client.27331 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 20 client.27331 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef65260 0x562f0ef67680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 4 client.27331 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x562f0ec95cf0 con 0x562f0ef65260 2024-04-06T14:19:58.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.267+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef65260 0x562f0ef67680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b3f7fe700 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef65260 0x562f0ef67680 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1fd50572b55dce8c server_cookie=c74bc76a62f68134 in_seq=0 out_seq=0 2024-04-06T14:19:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b1effd700 10 client.27331 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:58.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b1effd700 1 -- 172.21.15.90:0/975342298 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x562f0ec95cf0 con 0x562f0ef65260 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b1effd700 10 client.27331 handle_command_reply: tid=0 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 client.27331 shutdown 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 msgr2=0x562f0ef05660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f5b34010b20 tx=0x7f5b34015000 comp rx=0 tx=0).stop 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 shutdown_connections 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x562f0eeffde0 0x562f0ef05ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x562f0eef5780 0x562f0ef05120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x562f0ef65260 0x562f0ef67680 crc :-1 s=READY pgs=244 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 --2- 172.21.15.90:0/975342298 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x562f0eef60a0 0x562f0ef05660 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 >> 172.21.15.90:0/975342298 conn(0x562f0ee546a0 msgr2=0x562f0eef4770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b1effd700 0 client.27331 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 shutdown_connections 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 1 -- 172.21.15.90:0/975342298 wait complete. 2024-04-06T14:19:58.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.268+0000 7f5b5229db80 20 client.27331 trim_cache size 0 max 16384 2024-04-06T14:19:58.270 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 msgr2=0x7f5b280a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 0x7f5b280a3750 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f5b30009d20 tx=0x7f5b30009260 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 msgr2=0x7f5b4019e4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 0x7f5b4019e4e0 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f5b3400e9c0 tx=0x7f5b3400ecd0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 shutdown_connections 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b280a1290 0x7f5b280a3750 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b40109b60 0x7f5b40109ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b400ffe70 0x7f5b4019df80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 --2- 172.21.15.90:0/3047605771 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b40104f70 0x7f5b4019e4e0 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 >> 172.21.15.90:0/3047605771 conn(0x7f5b40076d50 msgr2=0x7f5b4010c1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:58.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.269+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 shutdown_connections 2024-04-06T14:19:58.272 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:58.270+0000 7f5b460e9700 1 -- 172.21.15.90:0/3047605771 wait complete. 2024-04-06T14:19:58.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:58 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3932464382' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:58.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:58 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3932464382' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:58.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:58 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3932464382' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:19:59.306 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:19:59.306 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:19:59.306 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:19:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:19:59 smithi105 ceph-mon[46969]: pgmap v456: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 810 op/s 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 -- 172.21.15.90:0/588506119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d4068aa0 msgr2=0x7f91d4105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 --2- 172.21.15.90:0/588506119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d4068aa0 0x7f91d4105970 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f91c8009990 tx=0x7f91c8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 -- 172.21.15.90:0/588506119 shutdown_connections 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 --2- 172.21.15.90:0/588506119 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d41060a0 0x7f91d410dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 --2- 172.21.15.90:0/588506119 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d40680f0 0x7f91d40684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.524 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 --2- 172.21.15.90:0/588506119 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d4068aa0 0x7f91d4105970 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 -- 172.21.15.90:0/588506119 >> 172.21.15.90:0/588506119 conn(0x7f91d4076d50 msgr2=0x7f91d4077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 -- 172.21.15.90:0/588506119 shutdown_connections 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.522+0000 7f91d8935700 1 -- 172.21.15.90:0/588506119 wait complete. 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 Processor -- start 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 -- start start 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 0x7f91d4197790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d419bb30 0x7f91d419bfb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91d419c5b0 con 0x7f91d40680f0 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91d419c720 con 0x7f91d41060a0 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d8935700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f91d419c8e0 con 0x7f91d419bb30 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.525 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 0x7f91d4197790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60880/0 (socket says 172.21.15.90:60880) 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d259c700 1 -- 172.21.15.90:0/321119509 learned_addr learned my addr 172.21.15.90:0/321119509 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d419bb30 0x7f91d419bfb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 -- 172.21.15.90:0/321119509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d419bb30 msgr2=0x7f91d419bfb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d419bb30 0x7f91d419bfb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 -- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 msgr2=0x7f91d4197250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d1d9b700 1 -- 172.21.15.90:0/321119509 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91c80095f0 con 0x7f91d41060a0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.523+0000 7f91d259c700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 0x7f91d4197790 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f91c80098e0 tx=0x7f91c8003a90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91c37fe700 1 -- 172.21.15.90:0/321119509 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c801d070 con 0x7f91d41060a0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91c37fe700 1 -- 172.21.15.90:0/321119509 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91c8005580 con 0x7f91d41060a0 2024-04-06T14:19:59.526 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f91d41a3de0 con 0x7f91d41060a0 2024-04-06T14:19:59.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91c37fe700 1 -- 172.21.15.90:0/321119509 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c800fbb0 con 0x7f91d41060a0 2024-04-06T14:19:59.527 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.524+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f91d41a41b0 con 0x7f91d41060a0 2024-04-06T14:19:59.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.527+0000 7f91c37fe700 1 -- 172.21.15.90:0/321119509 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f91c800fd10 con 0x7f91d41060a0 2024-04-06T14:19:59.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.527+0000 7f91c37fe700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 0x7f91bc0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.529 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.527+0000 7f91d259c700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 0x7f91bc0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.527+0000 7f91c37fe700 1 -- 172.21.15.90:0/321119509 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f91c80cdbf0 con 0x7f91d41060a0 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 Processor -- start 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 -- start start 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f906718be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d259c700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 0x7f91bc0a3750 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f91c4005950 tx=0x7f91c400b500 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906725250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f906670a10 con 0x55f906719120 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f90666e3d0 con 0x55f906718800 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f906669cc0 con 0x55f906722e60 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60892/0 (socket says 172.21.15.90:60892) 2024-04-06T14:19:59.530 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 -- 172.21.15.90:0/3797153310 learned_addr learned my addr 172.21.15.90:0/3797153310 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d259c700 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f906718be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906725250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 -- 172.21.15.90:0/3797153310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 msgr2=0x55f906725250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906725250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 -- 172.21.15.90:0/3797153310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 msgr2=0x55f906718be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f906718be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.528+0000 7f91d1d9b700 1 -- 172.21.15.90:0/3797153310 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f9064eb430 con 0x55f906719120 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f91c809aa70 tx=0x7f91c8003ed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=5ac3df89f8039f97 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91c0ff9700 1 -- 172.21.15.90:0/3797153310 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c809ad70 con 0x55f906719120 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91c0ff9700 1 -- 172.21.15.90:0/3797153310 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91c809eba0 con 0x55f906719120 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91c0ff9700 1 -- 172.21.15.90:0/3797153310 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c809b230 con 0x55f906719120 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/3797153310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 msgr2=0x55f906722920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 secure :-1 s=READY pgs=861 cs=0 l=1 rev1=1 crypto rx=0x7f91c809aa70 tx=0x7f91c8003ed0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.531 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/3797153310 shutdown_connections 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906725250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f906718be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/3797153310 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906722920 unknown :-1 s=CLOSED pgs=861 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/3797153310 >> 172.21.15.90:0/3797153310 conn(0x55f906677720 msgr2=0x55f906713b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/3797153310 shutdown_connections 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.529+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/3797153310 wait complete. 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.530+0000 7f91e4ae9b80 1 -- start start 2024-04-06T14:19:59.532 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.530+0000 7f91e4ae9b80 1 Processor -- start 2024-04-06T14:19:59.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067224e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f90671b790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f906605e00 con 0x55f906722e60 2024-04-06T14:19:59.533 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f906605aa0 con 0x55f906718800 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f9064ed720 con 0x55f906719120 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91e4ae9b80 10 client.0 _ll_register_callbacks cb 0x55f90660f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.532+0000 7f91d2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60898/0 (socket says 172.21.15.90:60898) 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 -- 172.21.15.90:0/4011725462 learned_addr learned my addr 172.21.15.90:0/4011725462 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f90671b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d259c700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067224e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 -- 172.21.15.90:0/4011725462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 msgr2=0x55f90671b790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f90671b790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 -- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 msgr2=0x55f9067224e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067224e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.534 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 -- 172.21.15.90:0/4011725462 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f91d41a41b0 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d259c700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067224e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 secure :-1 s=READY pgs=862 cs=0 l=1 rev1=1 crypto rx=0x7f91cc004090 tx=0x7f91cc0040c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8ad147e5384b5121 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91b9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91cc004db0 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f91cc007500 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55f90662a400 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f91cc007700 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91e4ae9b80 10 client.17880 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.533+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f906497be0 con 0x55f906722e60 2024-04-06T14:19:59.535 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.534+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55f90662a400 con 0x55f906722e60 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f91cc007860 con 0x55f906722e60 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 10 client.17880 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 10 client.17880 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 20 client.17880 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f90674bf40 0x55f90674e330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 4 client.17880 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55f906662780 con 0x55f90674bf40 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91d259c700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f90674bf40 0x55f90674e330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.537 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91b9ffb700 10 client.17880 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.535+0000 7f91d259c700 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f90674bf40 0x55f90674e330 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1387d937296c4495 server_cookie=5086488c7df2374f in_seq=0 out_seq=0 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.536+0000 7f91b9ffb700 1 -- 172.21.15.90:0/4011725462 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55f906662780 con 0x55f90674bf40 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.536+0000 7f91b9ffb700 10 client.17880 handle_command_reply: tid=0 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.536+0000 7f91e4ae9b80 1 client.17880 shutdown 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.536+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 msgr2=0x55f90671bcd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.536+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 secure :-1 s=READY pgs=862 cs=0 l=1 rev1=1 crypto rx=0x7f91cc004090 tx=0x7f91cc0040c0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 shutdown_connections 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906722e60 0x55f90671bcd0 unknown :-1 s=CLOSED pgs=862 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067224e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f90674bf40 0x55f90674e330 crc :-1 s=READY pgs=245 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4011725462 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f90671b790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 >> 172.21.15.90:0/4011725462 conn(0x55f906677720 msgr2=0x55f906712ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.538 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91b9ffb700 0 client.17880 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 shutdown_connections 2024-04-06T14:19:59.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.537+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4011725462 wait complete. 2024-04-06T14:19:59.539 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.538+0000 7f91e4ae9b80 20 client.17880 trim_cache size 0 max 16384 2024-04-06T14:19:59.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:19:59 smithi090 ceph-mon[43420]: pgmap v456: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 810 op/s 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.557+0000 7f91e4ae9b80 1 Processor -- start 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.557+0000 7f91e4ae9b80 1 -- start start 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.557+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067221a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.557+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906713960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.558 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f9067146d0 con 0x55f906719120 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f9067197f0 con 0x55f906718800 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f90671cd90 con 0x55f906722e60 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60906/0 (socket says 172.21.15.90:60906) 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 -- 172.21.15.90:0/4259085586 learned_addr learned my addr 172.21.15.90:0/4259085586 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d259c700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067221a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906713960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 -- 172.21.15.90:0/4259085586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 msgr2=0x55f906713960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906713960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 -- 172.21.15.90:0/4259085586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 msgr2=0x55f9067221a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067221a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 -- 172.21.15.90:0/4259085586 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f90662ff20 con 0x55f906719120 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d259c700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067221a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:19:59.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f91c800c010 tx=0x7f91c8009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a21e4d22de3da6f4 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91c17fa700 1 -- 172.21.15.90:0/4259085586 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c809ad70 con 0x55f906719120 2024-04-06T14:19:59.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91c17fa700 1 -- 172.21.15.90:0/4259085586 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91c80a3160 con 0x55f906719120 2024-04-06T14:19:59.560 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.558+0000 7f91c17fa700 1 -- 172.21.15.90:0/4259085586 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c809a4c0 con 0x55f906719120 2024-04-06T14:19:59.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4259085586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 msgr2=0x55f906713420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.564 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 secure :-1 s=READY pgs=863 cs=0 l=1 rev1=1 crypto rx=0x7f91c800c010 tx=0x7f91c8009310 comp rx=0 tx=0).stop 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4259085586 shutdown_connections 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906722e60 0x55f906713960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906718800 0x55f9067221a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/4259085586 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906719120 0x55f906713420 unknown :-1 s=CLOSED pgs=863 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.559+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4259085586 >> 172.21.15.90:0/4259085586 conn(0x55f906677720 msgr2=0x55f906725d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4259085586 shutdown_connections 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/4259085586 wait complete. 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- start start 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 Processor -- start 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f906732d70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.565 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906722e60 0x55f90679fd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f906733550 con 0x55f906718800 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f9067336c0 con 0x55f906722e60 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f9067a02c0 con 0x55f906719120 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91e4ae9b80 10 client.0 _ll_register_callbacks cb 0x55f90660f4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91d259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60918/0 (socket says 172.21.15.90:60918) 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91d1d9b700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f906732d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.560+0000 7f91d259c700 1 -- 172.21.15.90:0/855746546 learned_addr learned my addr 172.21.15.90:0/855746546 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d2d9d700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906722e60 0x55f90679fd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 -- 172.21.15.90:0/855746546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 msgr2=0x55f906732d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f906732d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.566 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 -- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906722e60 msgr2=0x55f90679fd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906722e60 0x55f90679fd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 -- 172.21.15.90:0/855746546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55f9064eb430 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91d259c700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f91c4026380 tx=0x7f91c4013000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=136460bc7884c188 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91b9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91c4004020 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f91c4017070 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f91c809eba0 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f91c4004180 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91e4ae9b80 10 client.17886 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55f9064efd50 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.561+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91c809eba0 con 0x55f906718800 2024-04-06T14:19:59.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f91c4003680 con 0x55f906718800 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 10 client.17886 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 10 client.17886 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 20 client.17886 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f9067a1c00 0x55f9067a4020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 4 client.17886 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55f9064b8d70 con 0x55f9067a1c00 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f9067a1c00 0x55f9067a4020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91b9ffb700 10 client.17886 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.562+0000 7f91d1d9b700 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f9067a1c00 0x55f9067a4020 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b45d9670070d9fd server_cookie=454901ec63179fe2 in_seq=0 out_seq=0 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91b9ffb700 1 -- 172.21.15.90:0/855746546 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55f9064b8d70 con 0x55f9067a1c00 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91b9ffb700 10 client.17886 handle_command_reply: tid=0 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 client.17886 shutdown 2024-04-06T14:19:59.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 msgr2=0x55f906732830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 secure :-1 s=READY pgs=864 cs=0 l=1 rev1=1 crypto rx=0x7f91c4026380 tx=0x7f91c4013000 comp rx=0 tx=0).stop 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 shutdown_connections 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55f906722e60 0x55f90679fd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55f906718800 0x55f906732830 unknown :-1 s=CLOSED pgs=864 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55f9067a1c00 0x55f9067a4020 crc :-1 s=READY pgs=246 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 --2- 172.21.15.90:0/855746546 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55f906719120 0x55f906732d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 >> 172.21.15.90:0/855746546 conn(0x55f906677720 msgr2=0x55f906718080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91b9ffb700 0 client.17886 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.563+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 shutdown_connections 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.564+0000 7f91e4ae9b80 1 -- 172.21.15.90:0/855746546 wait complete. 2024-04-06T14:19:59.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.564+0000 7f91e4ae9b80 20 client.17886 trim_cache size 0 max 16384 2024-04-06T14:19:59.570 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:59.570 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:19:59.570 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "f881b8a0-07df-4fda-8880-ca3516950644", 2024-04-06T14:19:59.570 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:19:59.570 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 msgr2=0x7f91bc0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 0x7f91bc0a3750 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f91c4005950 tx=0x7f91c400b500 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 msgr2=0x7f91d4197790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 0x7f91d4197790 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f91c80098e0 tx=0x7f91c8003a90 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 shutdown_connections 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f91bc0a1290 0x7f91bc0a3750 secure :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f91c4005950 tx=0x7f91c400b500 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f91d419bb30 0x7f91d419bfb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f91d40680f0 0x7f91d4197250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 --2- 172.21.15.90:0/321119509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f91d41060a0 0x7f91d4197790 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 >> 172.21.15.90:0/321119509 conn(0x7f91d4076d50 msgr2=0x7f91d40fdf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 shutdown_connections 2024-04-06T14:19:59.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.565+0000 7f91d8935700 1 -- 172.21.15.90:0/321119509 wait complete. 2024-04-06T14:19:59.579 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:19:59.579 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:19:59.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:19:59 smithi134 ceph-mon[47549]: pgmap v456: 97 pgs: 97 active+clean; 6.1 GiB data, 24 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 16 MiB/s wr, 810 op/s 2024-04-06T14:19:59.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 -- 172.21.15.90:0/1451051318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 msgr2=0x7f06f410bf70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 --2- 172.21.15.90:0/1451051318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f410bf70 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f06dc005ce0 tx=0x7f06dc009a60 comp rx=0 tx=0).stop 2024-04-06T14:19:59.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 -- 172.21.15.90:0/1451051318 shutdown_connections 2024-04-06T14:19:59.847 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 --2- 172.21.15.90:0/1451051318 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 0x7f06f40790c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 --2- 172.21.15.90:0/1451051318 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 0x7f06f4108400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 --2- 172.21.15.90:0/1451051318 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f410bf70 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 -- 172.21.15.90:0/1451051318 >> 172.21.15.90:0/1451051318 conn(0x7f06f406be00 msgr2=0x7f06f406c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 -- 172.21.15.90:0/1451051318 shutdown_connections 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.835+0000 7f06f8c88700 1 -- 172.21.15.90:0/1451051318 wait complete. 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 Processor -- start 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 -- start start 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 0x7f06f4083850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 0x7f06f407de30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06f407e600 con 0x7f06f410c6a0 2024-04-06T14:19:59.848 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06f407e770 con 0x7f06f4108020 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.836+0000 7f06f8c88700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f06f41361e0 con 0x7f06f4108940 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55332/0 (socket says 172.21.15.90:55332) 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 -- 172.21.15.90:0/269048971 learned_addr learned my addr 172.21.15.90:0/269048971 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 0x7f06f407de30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06f259c700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 0x7f06f4083850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 -- 172.21.15.90:0/269048971 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 msgr2=0x7f06f4083850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 0x7f06f4083850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 -- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 msgr2=0x7f06f407de30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 0x7f06f407de30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.849 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 -- 172.21.15.90:0/269048971 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f06dc0095f0 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.837+0000 7f06ebfff700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f06dc00b580 tx=0x7f06dc00bee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.838+0000 7f06eb7fe700 1 -- 172.21.15.90:0/269048971 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06dc004a30 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.838+0000 7f06eb7fe700 1 -- 172.21.15.90:0/269048971 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06dc0054f0 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.838+0000 7f06eb7fe700 1 -- 172.21.15.90:0/269048971 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06dc00ea10 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.838+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f06f4136410 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.838+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06f4136870 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.841+0000 7f06eb7fe700 1 -- 172.21.15.90:0/269048971 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f06dc004c90 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.842+0000 7f06eb7fe700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 0x7f06e00a37c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.842+0000 7f0704e3cb80 1 Processor -- start 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f06eb7fe700 1 -- 172.21.15.90:0/269048971 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f06dc012070 con 0x7f06f4108940 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 -- start start 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426668b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.850 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b4266728a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4265c0990 con 0x55b426672de0 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4265be350 con 0x55b426668780 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.843+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4265b9c40 con 0x55b4266690a0 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.844+0000 7f06f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.844+0000 7f06f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60942/0 (socket says 172.21.15.90:60942) 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.844+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3029244499 learned_addr learned my addr 172.21.15.90:0/3029244499 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.845+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3029244499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 msgr2=0x55b4266728a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.845+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b4266728a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.851 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.845+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3029244499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 msgr2=0x55b426668b60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.845+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426668b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.845+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3029244499 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b42643b3b0 con 0x55b426672de0 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.846+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7f06ec00a430 tx=0x7f06ec009650 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d22ffb36cf6f2afd server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.846+0000 7f06e8ff9700 1 -- 172.21.15.90:0/3029244499 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06ec010040 con 0x55b426672de0 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.846+0000 7f06e8ff9700 1 -- 172.21.15.90:0/3029244499 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06ec013a50 con 0x55b426672de0 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.846+0000 7f06e8ff9700 1 -- 172.21.15.90:0/3029244499 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06ec014b60 con 0x55b426672de0 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.847+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3029244499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 msgr2=0x55b4266751d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.852 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.847+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 secure :-1 s=READY pgs=865 cs=0 l=1 rev1=1 crypto rx=0x7f06ec00a430 tx=0x7f06ec009650 comp rx=0 tx=0).stop 2024-04-06T14:19:59.853 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.849+0000 7f06f259c700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 0x7f06e00a37c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.856+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3029244499 shutdown_connections 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.856+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b4266751d0 unknown :-1 s=CLOSED pgs=865 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.856+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426668b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.856+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3029244499 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b4266728a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.856+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3029244499 >> 172.21.15.90:0/3029244499 conn(0x55b4265c76a0 msgr2=0x55b426663a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.863+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3029244499 shutdown_connections 2024-04-06T14:19:59.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.863+0000 7f06f259c700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 0x7f06e00a37c0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f06e400b870 tx=0x7f06e4008040 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.868+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3029244499 wait complete. 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.868+0000 7f0704e3cb80 1 -- start start 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.868+0000 7f0704e3cb80 1 Processor -- start 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b4266727e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b42666fb70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426670180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b426555d80 con 0x55b426672de0 2024-04-06T14:19:59.875 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b426555a20 con 0x55b426668780 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42643d6a0 con 0x55b4266690a0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.869+0000 7f0704e3cb80 10 client.0 _ll_register_callbacks cb 0x55b42655f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.870+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b42666fb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.870+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b42666fb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:55358/0 (socket says 172.21.15.90:55358) 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.870+0000 7f06ebfff700 1 -- 172.21.15.90:0/1303872921 learned_addr learned my addr 172.21.15.90:0/1303872921 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.870+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426670180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 -- 172.21.15.90:0/1303872921 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 msgr2=0x55b42666fb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b42666fb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 -- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 msgr2=0x55b4266727e0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b4266727e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 -- 172.21.15.90:0/1303872921 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06ec009c60 con 0x55b426672de0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426670180 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7f06ec004b10 tx=0x7f06ec00aeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6eccb891c9a1ca1a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.876 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06d9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06ec010070 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f06ec006e40 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.871+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b42657a380 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f06ec013790 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 10 client.17895 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b4263e7b60 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06ec02a030 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f06ec024060 con 0x55b426672de0 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 10 client.17895 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 10 client.17895 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 20 client.17895 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:59.877 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266828d0 0x55b426684cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 4 client.17895 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.872+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b4265b2700 con 0x55b4266828d0 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.873+0000 7f06f259c700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266828d0 0x55b426684cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.873+0000 7f06d9ffb700 10 client.17895 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.873+0000 7f06f259c700 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266828d0 0x55b426684cc0 crc :-1 s=READY pgs=247 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=aed700c70f8ecf0e server_cookie=1daab8206f923afa in_seq=0 out_seq=0 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.874+0000 7f06d9ffb700 1 -- 172.21.15.90:0/1303872921 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55b4265b2700 con 0x55b4266828d0 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.874+0000 7f06d9ffb700 10 client.17895 handle_command_reply: tid=0 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.874+0000 7f0704e3cb80 1 client.17895 shutdown 2024-04-06T14:19:59.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 msgr2=0x55b426670180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426670180 secure :-1 s=READY pgs=866 cs=0 l=1 rev1=1 crypto rx=0x7f06ec004b10 tx=0x7f06ec00aeb0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 shutdown_connections 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426670180 unknown :-1 s=CLOSED pgs=866 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b4266727e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266828d0 0x55b426684cc0 crc :-1 s=READY pgs=247 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/1303872921 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b4266690a0 0x55b42666fb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 >> 172.21.15.90:0/1303872921 conn(0x55b4265c76a0 msgr2=0x55b426662c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.879 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f06d9ffb700 0 client.17895 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 shutdown_connections 2024-04-06T14:19:59.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 1 -- 172.21.15.90:0/1303872921 wait complete. 2024-04-06T14:19:59.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.875+0000 7f0704e3cb80 20 client.17895 trim_cache size 0 max 16384 2024-04-06T14:19:59.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 Processor -- start 2024-04-06T14:19:59.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 -- start start 2024-04-06T14:19:59.903 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426679ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426672de0 0x55b42666e980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b426669a10 con 0x55b4266690a0 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b42666eec0 con 0x55b426668780 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b426664dc0 con 0x55b426672de0 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f06ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60960/0 (socket says 172.21.15.90:60960) 2024-04-06T14:19:59.904 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f06ebfff700 1 -- 172.21.15.90:0/133945989 learned_addr learned my addr 172.21.15.90:0/133945989 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f06f259c700 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426679ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.895+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426672de0 0x55b42666e980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 -- 172.21.15.90:0/133945989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426672de0 msgr2=0x55b42666e980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426672de0 0x55b42666e980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 -- 172.21.15.90:0/133945989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 msgr2=0x55b426679ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426679ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 -- 172.21.15.90:0/133945989 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b42657fea0 con 0x55b4266690a0 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06ebfff700 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f06dc0dbef0 tx=0x7f06dc009a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c4839f2da883a5b7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06e97fa700 1 -- 172.21.15.90:0/133945989 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06dc0df250 con 0x55b4266690a0 2024-04-06T14:19:59.905 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06e97fa700 1 -- 172.21.15.90:0/133945989 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06dc0defc0 con 0x55b4266690a0 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.896+0000 7f06e97fa700 1 -- 172.21.15.90:0/133945989 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06dc0e32b0 con 0x55b4266690a0 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- 172.21.15.90:0/133945989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 msgr2=0x55b42666e440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 secure :-1 s=READY pgs=867 cs=0 l=1 rev1=1 crypto rx=0x7f06dc0dbef0 tx=0x7f06dc009a60 comp rx=0 tx=0).stop 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- 172.21.15.90:0/133945989 shutdown_connections 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426672de0 0x55b42666e980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b426668780 0x55b426679ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/133945989 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b4266690a0 0x55b42666e440 unknown :-1 s=CLOSED pgs=867 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- 172.21.15.90:0/133945989 >> 172.21.15.90:0/133945989 conn(0x55b4265c76a0 msgr2=0x55b42666fb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.906 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- 172.21.15.90:0/133945989 shutdown_connections 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- 172.21.15.90:0/133945989 wait complete. 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 -- start start 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.897+0000 7f0704e3cb80 1 Processor -- start 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426668780 0x55b426678130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b4266690a0 0x55b426678670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4266d6680 con 0x55b426672de0 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4266d67c0 con 0x55b4266690a0 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b4266d6900 con 0x55b426668780 2024-04-06T14:19:59.907 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f0704e3cb80 10 client.0 _ll_register_callbacks cb 0x55b42655f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:19:59.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60966/0 (socket says 172.21.15.90:60966) 2024-04-06T14:19:59.908 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3048293038 learned_addr learned my addr 172.21.15.90:0/3048293038 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06ebfff700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b4266690a0 0x55b426678670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f259c700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426668780 0x55b426678130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3048293038 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426668780 msgr2=0x55b426678130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426668780 0x55b426678130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b4266690a0 msgr2=0x55b426678670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b4266690a0 0x55b426678670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 -- 172.21.15.90:0/3048293038 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f06f4136870 con 0x55b426672de0 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06f2d9d700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7f06ec009440 tx=0x7f06ec0137c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f8b11fea068bc8df server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:19:59.909 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.898+0000 7f06d9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f06ec010070 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(100..100 src has 1..100) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f06ec01eb40 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f06ec013a50 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f06ec014630 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 10 client.17901 fetch_fsmap learned FSMap version 24 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55b42643fcd0 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f06ec013a50 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f06ec036020 con 0x55b426672de0 2024-04-06T14:19:59.910 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 10 client.17901 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 10 client.17901 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 20 client.17901 populate_metadata read hostname 'smithi090' 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266d7cb0 0x55b4266da0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 4 client.17901 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.899+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55b426408cf0 con 0x55b4266d7cb0 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06f259c700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266d7cb0 0x55b4266da0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06f259c700 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266d7cb0 0x55b4266da0d0 crc :-1 s=READY pgs=248 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=66a23b3daf3749cc server_cookie=af6ddc2f6d70b55e in_seq=0 out_seq=0 2024-04-06T14:19:59.911 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06d9ffb700 10 client.17901 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06d9ffb700 1 -- 172.21.15.90:0/3048293038 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+266 (crc 0 0 0) 0x55b426408cf0 con 0x55b4266d7cb0 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06d9ffb700 10 client.17901 handle_command_reply: tid=0 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 client.17901 shutdown 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 msgr2=0x55b426678bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 secure :-1 s=READY pgs=868 cs=0 l=1 rev1=1 crypto rx=0x7f06ec009440 tx=0x7f06ec0137c0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 shutdown_connections 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55b426672de0 0x55b426678bb0 unknown :-1 s=CLOSED pgs=868 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55b4266d7cb0 0x55b4266da0d0 crc :-1 s=READY pgs=248 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55b426668780 0x55b426678130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.912 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 --2- 172.21.15.90:0/3048293038 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55b4266690a0 0x55b426678670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 >> 172.21.15.90:0/3048293038 conn(0x55b4265c76a0 msgr2=0x55b42666fb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.900+0000 7f06d9ffb700 0 client.17901 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 shutdown_connections 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f0704e3cb80 1 -- 172.21.15.90:0/3048293038 wait complete. 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f0704e3cb80 20 client.17901 trim_cache size 0 max 16384 2024-04-06T14:19:59.913 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (38 inodes in the stack)", 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "f881b8a0-07df-4fda-8880-ca3516950644": { 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "f881b8a0-07df-4fda-8880-ca3516950644", 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:19:59.914 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:19:59.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 msgr2=0x7f06e00a37c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 0x7f06e00a37c0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f06e400b870 tx=0x7f06e4008040 comp rx=0 tx=0).stop 2024-04-06T14:19:59.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 msgr2=0x7f06f407d8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f06dc00b580 tx=0x7f06dc00bee0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 shutdown_connections 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f06e00a1300 0x7f06e00a37c0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f06f410c6a0 0x7f06f407de30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f4108020 0x7f06f4083850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 --2- 172.21.15.90:0/269048971 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f06f4108940 0x7f06f407d8f0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 >> 172.21.15.90:0/269048971 conn(0x7f06f406be00 msgr2=0x7f06f4070b40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.901+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 shutdown_connections 2024-04-06T14:19:59.916 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:19:59.902+0000 7f06f8c88700 1 -- 172.21.15.90:0/269048971 wait complete. 2024-04-06T14:19:59.917 INFO:tasks.cephfs.filesystem:scrub status for tag:f881b8a0-07df-4fda-8880-ca3516950644 - {'path': '/', 'tag': 'f881b8a0-07df-4fda-8880-ca3516950644', 'options': 'recursive,force'} 2024-04-06T14:20:00.189 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:00 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:20:00.189 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:00 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.189 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:00 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:20:00.189 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:00 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.189 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:00 smithi090 ceph-mon[43420]: overall HEALTH_OK 2024-04-06T14:20:00.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:00 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:20:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:00 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:00 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:20:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:00 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:00 smithi105 ceph-mon[46969]: overall HEALTH_OK 2024-04-06T14:20:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:00 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:20:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:00 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:00 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:20:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:00 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:00.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:00 smithi134 ceph-mon[47549]: overall HEALTH_OK 2024-04-06T14:20:01.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:01 smithi105 ceph-mon[46969]: pgmap v457: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 12 MiB/s wr, 573 op/s 2024-04-06T14:20:01.500 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:01 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:20:01.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:01 smithi090 ceph-mon[43420]: pgmap v457: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 12 MiB/s wr, 573 op/s 2024-04-06T14:20:01.542 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:01 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:20:01.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:01 smithi134 ceph-mon[47549]: pgmap v457: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 12 MiB/s wr, 573 op/s 2024-04-06T14:20:01.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:01 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:20:03.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 855.0 s, 1013.4 tps, lat 97.220 ms stddev 125.186 2024-04-06T14:20:03.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:03 smithi105 ceph-mon[46969]: pgmap v458: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 14 MiB/s wr, 835 op/s 2024-04-06T14:20:03.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:03 smithi090 ceph-mon[43420]: pgmap v458: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 14 MiB/s wr, 835 op/s 2024-04-06T14:20:03.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:03 smithi134 ceph-mon[47549]: pgmap v458: 97 pgs: 97 active+clean; 6.1 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 40 KiB/s rd, 14 MiB/s wr, 835 op/s 2024-04-06T14:20:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:05 smithi105 ceph-mon[46969]: pgmap v459: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:20:05.550 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:05 smithi090 ceph-mon[43420]: pgmap v459: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:20:05.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:05 smithi134 ceph-mon[47549]: pgmap v459: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 140 KiB/s rd, 19 MiB/s wr, 1.24k op/s 2024-04-06T14:20:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:07 smithi090 ceph-mon[43420]: pgmap v460: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 17 MiB/s wr, 1.26k op/s 2024-04-06T14:20:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:07 smithi090 ceph-mon[43420]: osdmap e101: 12 total, 12 up, 12 in 2024-04-06T14:20:07.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:07 smithi134 ceph-mon[47549]: pgmap v460: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 17 MiB/s wr, 1.26k op/s 2024-04-06T14:20:07.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:07 smithi134 ceph-mon[47549]: osdmap e101: 12 total, 12 up, 12 in 2024-04-06T14:20:07.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:07 smithi105 ceph-mon[46969]: pgmap v460: 97 pgs: 97 active+clean; 6.2 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 134 KiB/s rd, 17 MiB/s wr, 1.26k op/s 2024-04-06T14:20:07.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:07 smithi105 ceph-mon[46969]: osdmap e101: 12 total, 12 up, 12 in 2024-04-06T14:20:08.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 860.0 s, 1020.2 tps, lat 98.917 ms stddev 124.607 2024-04-06T14:20:09.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:09 smithi090 ceph-mon[43420]: pgmap v462: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 21 MiB/s wr, 1.74k op/s 2024-04-06T14:20:09.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:09 smithi134 ceph-mon[47549]: pgmap v462: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 21 MiB/s wr, 1.74k op/s 2024-04-06T14:20:09.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:09 smithi105 ceph-mon[46969]: pgmap v462: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 21 MiB/s wr, 1.74k op/s 2024-04-06T14:20:11.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:11 smithi105 ceph-mon[46969]: pgmap v463: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 22 MiB/s wr, 1.78k op/s 2024-04-06T14:20:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:11 smithi090 ceph-mon[43420]: pgmap v463: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 22 MiB/s wr, 1.78k op/s 2024-04-06T14:20:11.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:11 smithi134 ceph-mon[47549]: pgmap v463: 97 pgs: 7 active+clean+snaptrim, 8 active+clean+snaptrim_wait, 82 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 174 KiB/s rd, 22 MiB/s wr, 1.78k op/s 2024-04-06T14:20:13.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 865.0 s, 1952.3 tps, lat 51.593 ms stddev 63.112 2024-04-06T14:20:13.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:13 smithi105 ceph-mon[46969]: pgmap v464: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 21 MiB/s wr, 1.53k op/s 2024-04-06T14:20:13.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:13 smithi105 ceph-mon[46969]: osdmap e102: 12 total, 12 up, 12 in 2024-04-06T14:20:13.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:13 smithi090 ceph-mon[43420]: pgmap v464: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 21 MiB/s wr, 1.53k op/s 2024-04-06T14:20:13.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:13 smithi090 ceph-mon[43420]: osdmap e102: 12 total, 12 up, 12 in 2024-04-06T14:20:13.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:13 smithi134 ceph-mon[47549]: pgmap v464: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 163 KiB/s rd, 21 MiB/s wr, 1.53k op/s 2024-04-06T14:20:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:13 smithi134 ceph-mon[47549]: osdmap e102: 12 total, 12 up, 12 in 2024-04-06T14:20:15.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:15 smithi105 ceph-mon[46969]: pgmap v466: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:20:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:15 smithi090 ceph-mon[43420]: pgmap v466: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:20:15.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:15 smithi134 ceph-mon[47549]: pgmap v466: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 1.2 MiB/s rd, 18 MiB/s wr, 1.13k op/s 2024-04-06T14:20:17.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:17 smithi105 ceph-mon[46969]: pgmap v467: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 16 MiB/s wr, 959 op/s 2024-04-06T14:20:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:17 smithi090 ceph-mon[43420]: pgmap v467: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 16 MiB/s wr, 959 op/s 2024-04-06T14:20:17.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:17 smithi134 ceph-mon[47549]: pgmap v467: 97 pgs: 97 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 978 KiB/s rd, 16 MiB/s wr, 959 op/s 2024-04-06T14:20:18.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 870.0 s, 1350.8 tps, lat 74.223 ms stddev 80.486 2024-04-06T14:20:19.296 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:19 smithi090 ceph-mon[43420]: pgmap v468: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 13 MiB/s wr, 676 op/s 2024-04-06T14:20:19.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:19 smithi134 ceph-mon[47549]: pgmap v468: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 13 MiB/s wr, 676 op/s 2024-04-06T14:20:19.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:19 smithi105 ceph-mon[46969]: pgmap v468: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 13 MiB/s wr, 676 op/s 2024-04-06T14:20:21.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:21 smithi105 ceph-mon[46969]: pgmap v469: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 12 MiB/s wr, 629 op/s 2024-04-06T14:20:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:21 smithi090 ceph-mon[43420]: pgmap v469: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 12 MiB/s wr, 629 op/s 2024-04-06T14:20:21.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:21 smithi134 ceph-mon[47549]: pgmap v469: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 12 MiB/s wr, 629 op/s 2024-04-06T14:20:23.025 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 875.0 s, 1172.6 tps, lat 84.967 ms stddev 120.223 2024-04-06T14:20:23.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:23 smithi090 ceph-mon[43420]: pgmap v470: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 14 MiB/s wr, 912 op/s 2024-04-06T14:20:23.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:23 smithi134 ceph-mon[47549]: pgmap v470: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 14 MiB/s wr, 912 op/s 2024-04-06T14:20:23.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:23 smithi105 ceph-mon[46969]: pgmap v470: 97 pgs: 97 active+clean; 5.6 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 954 KiB/s rd, 14 MiB/s wr, 912 op/s 2024-04-06T14:20:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:25 smithi090 ceph-mon[43420]: pgmap v471: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:20:25.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:25 smithi134 ceph-mon[47549]: pgmap v471: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:20:25.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:25 smithi105 ceph-mon[46969]: pgmap v471: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 60 KiB/s rd, 16 MiB/s wr, 1.27k op/s 2024-04-06T14:20:27.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:27 smithi090 ceph-mon[43420]: pgmap v472: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 1.34k op/s 2024-04-06T14:20:27.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:27 smithi134 ceph-mon[47549]: pgmap v472: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 1.34k op/s 2024-04-06T14:20:27.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:27 smithi105 ceph-mon[46969]: pgmap v472: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 17 MiB/s wr, 1.34k op/s 2024-04-06T14:20:28.027 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 880.0 s, 1617.4 tps, lat 61.887 ms stddev 70.337 2024-04-06T14:20:29.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:29 smithi134 ceph-mon[47549]: pgmap v473: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 20 MiB/s wr, 1.54k op/s 2024-04-06T14:20:29.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:29 smithi105 ceph-mon[46969]: pgmap v473: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 20 MiB/s wr, 1.54k op/s 2024-04-06T14:20:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:29 smithi090 ceph-mon[43420]: pgmap v473: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 71 KiB/s rd, 20 MiB/s wr, 1.54k op/s 2024-04-06T14:20:29.918 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:20:29.918 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 -- 172.21.15.90:0/2392938835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec500ff220 msgr2=0x7fec500ff6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 --2- 172.21.15.90:0/2392938835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec500ff220 0x7fec500ff6a0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7fec3c009990 tx=0x7fec3c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 -- 172.21.15.90:0/2392938835 shutdown_connections 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 --2- 172.21.15.90:0/2392938835 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 0x7fec50103960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 --2- 172.21.15.90:0/2392938835 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec50105e60 0x7fec50106240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 --2- 172.21.15.90:0/2392938835 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec500ff220 0x7fec500ff6a0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 -- 172.21.15.90:0/2392938835 >> 172.21.15.90:0/2392938835 conn(0x7fec500761b0 msgr2=0x7fec500fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 -- 172.21.15.90:0/2392938835 shutdown_connections 2024-04-06T14:20:30.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.216+0000 7fec56b13700 1 -- 172.21.15.90:0/2392938835 wait complete. 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.217+0000 7fec56b13700 1 Processor -- start 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 -- start start 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec500ff220 0x7fec500723f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 0x7fec50072970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5006bd90 con 0x7fec500ff220 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5006bf00 con 0x7fec500ffcc0 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec56b13700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec5006c0c0 con 0x7fec50105e60 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec550b0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec550b0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39032/0 (socket says 172.21.15.90:39032) 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec550b0700 1 -- 172.21.15.90:0/2365504597 learned_addr learned my addr 172.21.15.90:0/2365504597 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec548af700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec500ff220 0x7fec500723f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.218+0000 7fec4ffff700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 0x7fec50072970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 -- 172.21.15.90:0/2365504597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 msgr2=0x7fec50072970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 0x7fec50072970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 -- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec500ff220 msgr2=0x7fec500723f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec500ff220 0x7fec500723f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 -- 172.21.15.90:0/2365504597 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec3c0095f0 con 0x7fec50105e60 2024-04-06T14:20:30.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec550b0700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7fec4800ef30 tx=0x7fec4800c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec4dffb700 1 -- 172.21.15.90:0/2365504597 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec48018070 con 0x7fec50105e60 2024-04-06T14:20:30.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec5006c340 con 0x7fec50105e60 2024-04-06T14:20:30.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.219+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec50070300 con 0x7fec50105e60 2024-04-06T14:20:30.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.220+0000 7fec4dffb700 1 -- 172.21.15.90:0/2365504597 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec4800f040 con 0x7fec50105e60 2024-04-06T14:20:30.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.220+0000 7fec4dffb700 1 -- 172.21.15.90:0/2365504597 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec4800cd90 con 0x7fec50105e60 2024-04-06T14:20:30.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.232+0000 7fec62cc7b80 1 Processor -- start 2024-04-06T14:20:30.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.233+0000 7fec4dffb700 1 -- 172.21.15.90:0/2365504597 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fec48003a00 con 0x7fec50105e60 2024-04-06T14:20:30.257 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.233+0000 7fec62cc7b80 1 -- start start 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4298b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b38b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42b61e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c41f0990 con 0x5565c42b3df0 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c41ee350 con 0x5565c42990a0 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.262+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c41e9c40 con 0x5565c4298780 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.265+0000 7fec4ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4298b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.265+0000 7fec4ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4298b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39040/0 (socket says 172.21.15.90:39040) 2024-04-06T14:20:30.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.265+0000 7fec4ffff700 1 -- 172.21.15.90:0/1945237420 learned_addr learned my addr 172.21.15.90:0/1945237420 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.265+0000 7fec4dffb700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 0x7fec400a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.266+0000 7fec4dffb700 1 -- 172.21.15.90:0/2365504597 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fec480cc770 con 0x7fec50105e60 2024-04-06T14:20:30.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.268+0000 7fec4ffff700 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42b61e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.271+0000 7fec550b0700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 0x7fec400a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.271+0000 7fec550b0700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 0x7fec400a3890 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fec480c9ea0 tx=0x7fec480954b0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.273 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.273+0000 7fec548af700 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b38b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 -- 172.21.15.90:0/1945237420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 msgr2=0x5565c4298b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4298b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 -- 172.21.15.90:0/1945237420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 msgr2=0x5565c42b38b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b38b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 -- 172.21.15.90:0/1945237420 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5565c406b3b0 con 0x5565c42b3df0 2024-04-06T14:20:30.281 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.280+0000 7fec4ffff700 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42b61e0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7fec3c00b6f0 tx=0x7fec3c009ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dcffd8c84de6b6a3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.291+0000 7fec3affd700 1 -- 172.21.15.90:0/1945237420 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec3c00bda0 con 0x5565c42b3df0 2024-04-06T14:20:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.291+0000 7fec3affd700 1 -- 172.21.15.90:0/1945237420 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec3c01e070 con 0x5565c42b3df0 2024-04-06T14:20:30.291 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.291+0000 7fec3affd700 1 -- 172.21.15.90:0/1945237420 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec3c0286b0 con 0x5565c42b3df0 2024-04-06T14:20:30.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.296+0000 7fec62cc7b80 1 -- 172.21.15.90:0/1945237420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 msgr2=0x5565c42b61e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.296+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42b61e0 secure :-1 s=READY pgs=869 cs=0 l=1 rev1=1 crypto rx=0x7fec3c00b6f0 tx=0x7fec3c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.298+0000 7fec62cc7b80 1 -- 172.21.15.90:0/1945237420 shutdown_connections 2024-04-06T14:20:30.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.298+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42b61e0 unknown :-1 s=CLOSED pgs=869 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.298+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4298b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.298+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/1945237420 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b38b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.298 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.298+0000 7fec62cc7b80 1 -- 172.21.15.90:0/1945237420 >> 172.21.15.90:0/1945237420 conn(0x5565c41f76a0 msgr2=0x5565c4293a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.307+0000 7fec62cc7b80 1 -- 172.21.15.90:0/1945237420 shutdown_connections 2024-04-06T14:20:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.320+0000 7fec62cc7b80 1 -- 172.21.15.90:0/1945237420 wait complete. 2024-04-06T14:20:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.320+0000 7fec62cc7b80 1 -- start start 2024-04-06T14:20:30.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.320+0000 7fec62cc7b80 1 Processor -- start 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c4298780 0x5565c42b3880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b1360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c4185d80 con 0x5565c4298780 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c4185a20 con 0x5565c42b3df0 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c406d6a0 con 0x5565c42990a0 2024-04-06T14:20:30.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.323+0000 7fec62cc7b80 10 client.0 _ll_register_callbacks cb 0x5565c418f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39056/0 (socket says 172.21.15.90:39056) 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 -- 172.21.15.90:0/2042908159 learned_addr learned my addr 172.21.15.90:0/2042908159 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 -- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 msgr2=0x5565c42b1360 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b1360 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 -- 172.21.15.90:0/2042908159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c4298780 msgr2=0x5565c42b3880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c4298780 0x5565c42b3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 -- 172.21.15.90:0/2042908159 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec50104490 con 0x5565c42990a0 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec548af700 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fec44011290 tx=0x7fec44011650 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7a98db8da88ae70c server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec2ffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:20:30.326 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.324+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec44015660 con 0x5565c42990a0 2024-04-06T14:20:30.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fec440157c0 con 0x5565c42990a0 2024-04-06T14:20:30.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5565c41aa380 con 0x5565c42990a0 2024-04-06T14:20:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fec44015980 con 0x5565c42990a0 2024-04-06T14:20:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec62cc7b80 10 client.27376 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:30.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5565c4017b60 con 0x5565c42990a0 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.325+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5565c41aa380 con 0x5565c42990a0 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fec44014ca0 con 0x5565c42990a0 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 10 client.27376 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 10 client.27376 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 20 client.27376 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c42c3ae0 0x5565c42c5fa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 4 client.27376 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.326+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5565c41e2700 con 0x5565c42c3ae0 2024-04-06T14:20:30.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.327+0000 7fec4ffff700 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c42c3ae0 0x5565c42c5fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.328+0000 7fec2ffff700 10 client.27376 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.328+0000 7fec4ffff700 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c42c3ae0 0x5565c42c5fa0 crc :-1 s=READY pgs=250 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=398a012b407e5765 server_cookie=c170e2beaeac5d7a in_seq=0 out_seq=0 2024-04-06T14:20:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.328+0000 7fec2ffff700 1 -- 172.21.15.90:0/2042908159 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x5565c41e2700 con 0x5565c42c3ae0 2024-04-06T14:20:30.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.328+0000 7fec2ffff700 10 client.27376 handle_command_reply: tid=0 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 client.27376 shutdown 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 msgr2=0x5565c42b0c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7fec44011290 tx=0x7fec44011650 comp rx=0 tx=0).stop 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 shutdown_connections 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b1360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c4298780 0x5565c42b3880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c42c3ae0 0x5565c42c5fa0 crc :-1 s=READY pgs=250 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/2042908159 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c42990a0 0x5565c42b0c70 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 >> 172.21.15.90:0/2042908159 conn(0x5565c41f76a0 msgr2=0x5565c4292c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.329+0000 7fec2ffff700 0 client.27376 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.330+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 shutdown_connections 2024-04-06T14:20:30.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.330+0000 7fec62cc7b80 1 -- 172.21.15.90:0/2042908159 wait complete. 2024-04-06T14:20:30.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.330+0000 7fec62cc7b80 20 client.27376 trim_cache size 0 max 16384 2024-04-06T14:20:30.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.354+0000 7fec62cc7b80 1 Processor -- start 2024-04-06T14:20:30.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.354+0000 7fec62cc7b80 1 -- start start 2024-04-06T14:20:30.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4293c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.355 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42bf1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c42abaa0 con 0x5565c42b3df0 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c4298eb0 con 0x5565c42990a0 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c42af1a0 con 0x5565c4298780 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:37974/0 (socket says 172.21.15.90:37974) 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 -- 172.21.15.90:0/825489762 learned_addr learned my addr 172.21.15.90:0/825489762 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec4ffff700 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4293c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.356 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 -- 172.21.15.90:0/825489762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 msgr2=0x5565c4293c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4293c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 -- 172.21.15.90:0/825489762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 msgr2=0x5565c42bf1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42bf1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.355+0000 7fec548af700 1 -- 172.21.15.90:0/825489762 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5565c41afea0 con 0x5565c42990a0 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.356+0000 7fec4ffff700 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4293c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.356+0000 7fec548af700 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7fec44008010 tx=0x7fec44006b00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c09f969bd8367185 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.357 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.356+0000 7fec3b7fe700 1 -- 172.21.15.90:0/825489762 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec44005c50 con 0x5565c42990a0 2024-04-06T14:20:30.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec3b7fe700 1 -- 172.21.15.90:0/825489762 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec4401d650 con 0x5565c42990a0 2024-04-06T14:20:30.358 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec3b7fe700 1 -- 172.21.15.90:0/825489762 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec44006460 con 0x5565c42990a0 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 -- 172.21.15.90:0/825489762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 msgr2=0x5565c42b0530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7fec44008010 tx=0x7fec44006b00 comp rx=0 tx=0).stop 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 -- 172.21.15.90:0/825489762 shutdown_connections 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42b3df0 0x5565c42bf1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c4293c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.359 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/825489762 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42990a0 0x5565c42b0530 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.358+0000 7fec62cc7b80 1 -- 172.21.15.90:0/825489762 >> 172.21.15.90:0/825489762 conn(0x5565c41f76a0 msgr2=0x5565c4293520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- 172.21.15.90:0/825489762 shutdown_connections 2024-04-06T14:20:30.360 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- 172.21.15.90:0/825489762 wait complete. 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- start start 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 Processor -- start 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c42ba6f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 0x5565c42bac30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b8ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c42b93f0 con 0x5565c42990a0 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c42b9560 con 0x5565c42b3df0 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x5565c42b9720 con 0x5565c4298780 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.359+0000 7fec62cc7b80 10 client.0 _ll_register_callbacks cb 0x5565c418f470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec548af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 0x5565c42bac30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c42ba6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec548af700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 0x5565c42bac30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35248/0 (socket says 172.21.15.90:35248) 2024-04-06T14:20:30.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec548af700 1 -- 172.21.15.90:0/3927975179 learned_addr learned my addr 172.21.15.90:0/3927975179 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b8ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 -- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 msgr2=0x5565c42b8ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b8ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 -- 172.21.15.90:0/3927975179 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 msgr2=0x5565c42bac30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 0x5565c42bac30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.360+0000 7fec4ffff700 1 -- 172.21.15.90:0/3927975179 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5565c4017b60 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec4ffff700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c42ba6f0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fec3c00b2e0 tx=0x7fec3c009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8f071b2afaf739af server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec2ffff700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec3c02c040 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fec3c035500 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fec3c01e070 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fec3c00bba0 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec62cc7b80 10 client.27379 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x5565c406fcd0 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.361+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec3c01e070 con 0x5565c4298780 2024-04-06T14:20:30.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fec3c040030 con 0x5565c4298780 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 10 client.27379 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 10 client.27379 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 20 client.27379 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c43189b0 0x5565c431add0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 4 client.27379 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5565c4038cf0 con 0x5565c43189b0 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.362+0000 7fec548af700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c43189b0 0x5565c431add0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.363+0000 7fec548af700 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c43189b0 0x5565c431add0 crc :-1 s=READY pgs=251 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=22e2940e07429d21 server_cookie=4cf65064ca580017 in_seq=0 out_seq=0 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.363+0000 7fec2ffff700 10 client.27379 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.363+0000 7fec2ffff700 1 -- 172.21.15.90:0/3927975179 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x5565c4038cf0 con 0x5565c43189b0 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.363+0000 7fec2ffff700 10 client.27379 handle_command_reply: tid=0 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 client.27379 shutdown 2024-04-06T14:20:30.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 msgr2=0x5565c42ba6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c42ba6f0 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7fec3c00b2e0 tx=0x7fec3c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 shutdown_connections 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x5565c42b3df0 0x5565c42b8ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x5565c43189b0 0x5565c431add0 crc :-1 s=READY pgs=251 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x5565c4298780 0x5565c42ba6f0 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 --2- 172.21.15.90:0/3927975179 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x5565c42990a0 0x5565c42bac30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 >> 172.21.15.90:0/3927975179 conn(0x5565c41f76a0 msgr2=0x5565c4292390 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec2ffff700 0 client.27379 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 shutdown_connections 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 1 -- 172.21.15.90:0/3927975179 wait complete. 2024-04-06T14:20:30.367 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.364+0000 7fec62cc7b80 20 client.27379 trim_cache size 0 max 16384 2024-04-06T14:20:30.368 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:20:30.368 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:20:30.368 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:20:30.368 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 msgr2=0x7fec400a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 0x7fec400a3890 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fec480c9ea0 tx=0x7fec480954b0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 msgr2=0x7fec5006b5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7fec4800ef30 tx=0x7fec4800c3a0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 shutdown_connections 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fec400a13d0 0x7fec400a3890 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fec50105e60 0x7fec5006b5c0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fec500ff220 0x7fec500723f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 --2- 172.21.15.90:0/2365504597 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fec500ffcc0 0x7fec50072970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 >> 172.21.15.90:0/2365504597 conn(0x7fec500761b0 msgr2=0x7fec501014c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 shutdown_connections 2024-04-06T14:20:30.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.366+0000 7fec56b13700 1 -- 172.21.15.90:0/2365504597 wait complete. 2024-04-06T14:20:30.383 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:20:30.383 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 -- 172.21.15.90:0/3651673935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb6980ff4e0 msgr2=0x7fb6980ff8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 --2- 172.21.15.90:0/3651673935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb6980ff4e0 0x7fb6980ff8c0 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7fb688009990 tx=0x7fb688009ca0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 -- 172.21.15.90:0/3651673935 shutdown_connections 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 --2- 172.21.15.90:0/3651673935 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb698104600 0x7fb69810c180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 --2- 172.21.15.90:0/3651673935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb6980ff4e0 0x7fb6980ff8c0 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 --2- 172.21.15.90:0/3651673935 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb6980ffe90 0x7fb698103ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.579+0000 7fb69db5f700 1 -- 172.21.15.90:0/3651673935 >> 172.21.15.90:0/3651673935 conn(0x7fb6980760a0 msgr2=0x7fb6980fd450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:30.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.580+0000 7fb69db5f700 1 -- 172.21.15.90:0/3651673935 shutdown_connections 2024-04-06T14:20:30.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.580+0000 7fb69db5f700 1 -- 172.21.15.90:0/3651673935 wait complete. 2024-04-06T14:20:30.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.581+0000 7fb69db5f700 1 Processor -- start 2024-04-06T14:20:30.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.581+0000 7fb69db5f700 1 -- start start 2024-04-06T14:20:30.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6980ff4e0 0x7fb698072480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69806b580 0x7fb69806ba30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb69806c1c0 con 0x7fb6980ff4e0 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb69806c330 con 0x7fb69806b580 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb69db5f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb69806ffa0 con 0x7fb698104600 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39104/0 (socket says 172.21.15.90:39104) 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 -- 172.21.15.90:0/4166255419 learned_addr learned my addr 172.21.15.90:0/4166255419 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb6977fe700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6980ff4e0 0x7fb698072480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb697fff700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69806b580 0x7fb69806ba30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 -- 172.21.15.90:0/4166255419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69806b580 msgr2=0x7fb69806ba30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69806b580 0x7fb69806ba30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 -- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6980ff4e0 msgr2=0x7fb698072480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:30.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6980ff4e0 0x7fb698072480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 -- 172.21.15.90:0/4166255419 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6880095f0 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.582+0000 7fb696ffd700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fb68c00d980 tx=0x7fb68c00dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.583+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68c00f890 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.583+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb68c010460 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.583+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb68c00e610 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.583+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb698070280 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.583+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb698070740 con 0x7fb698104600 2024-04-06T14:20:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.584+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb698107870 con 0x7fb698104600 2024-04-06T14:20:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.585+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb68c0105d0 con 0x7fb698104600 2024-04-06T14:20:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.586+0000 7fb694ff9700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 0x7fb6800a3730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.586+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb68c0cc6e0 con 0x7fb698104600 2024-04-06T14:20:30.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.587+0000 7fb6977fe700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 0x7fb6800a3730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:30.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.587+0000 7fb6977fe700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 0x7fb6800a3730 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fb688006010 tx=0x7fb688005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:30.589 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:30.589+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb68c095b10 con 0x7fb698104600 2024-04-06T14:20:31.000 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.000+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb69806c9c0 con 0x7fb698104600 2024-04-06T14:20:31.001 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.001+0000 7fb694ff9700 1 -- 172.21.15.90:0/4166255419 <== mon.1 v2:172.21.15.134:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fb68c020090 con 0x7fb698104600 2024-04-06T14:20:31.001 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:31.001 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 msgr2=0x7fb6800a3730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 0x7fb6800a3730 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7fb688006010 tx=0x7fb688005960 comp rx=0 tx=0).stop 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 msgr2=0x7fb6980729c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fb68c00d980 tx=0x7fb68c00dc90 comp rx=0 tx=0).stop 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 shutdown_connections 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb6800a1270 0x7fb6800a3730 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69806b580 0x7fb69806ba30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.004 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb6980ff4e0 0x7fb698072480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 --2- 172.21.15.90:0/4166255419 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb698104600 0x7fb6980729c0 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 >> 172.21.15.90:0/4166255419 conn(0x7fb6980760a0 msgr2=0x7fb698102830 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 shutdown_connections 2024-04-06T14:20:31.005 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.003+0000 7fb69db5f700 1 -- 172.21.15.90:0/4166255419 wait complete. 2024-04-06T14:20:31.005 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:20:31.022 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:20:31.022 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.212+0000 7f415b417700 1 -- 172.21.15.90:0/2087432988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4154106090 msgr2=0x7f415410dc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.212+0000 7f415b417700 1 --2- 172.21.15.90:0/2087432988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4154106090 0x7f415410dc10 secure :-1 s=READY pgs=870 cs=0 l=1 rev1=1 crypto rx=0x7f414c00b0a0 tx=0x7f414c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 -- 172.21.15.90:0/2087432988 shutdown_connections 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 --2- 172.21.15.90:0/2087432988 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4154106090 0x7f415410dc10 unknown :-1 s=CLOSED pgs=870 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 --2- 172.21.15.90:0/2087432988 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f4154101350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 --2- 172.21.15.90:0/2087432988 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154101920 0x7f4154105960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 -- 172.21.15.90:0/2087432988 >> 172.21.15.90:0/2087432988 conn(0x7f41540fc7f0 msgr2=0x7f41540fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 -- 172.21.15.90:0/2087432988 shutdown_connections 2024-04-06T14:20:31.214 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.213+0000 7f415b417700 1 -- 172.21.15.90:0/2087432988 wait complete. 2024-04-06T14:20:31.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 Processor -- start 2024-04-06T14:20:31.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 -- start start 2024-04-06T14:20:31.216 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f415410cfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 0x7f415410d510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f415419de60 0x7f415419e2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f415419e8e0 con 0x7f415419de60 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f415419ea50 con 0x7f4154106090 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f415b417700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f415419ec10 con 0x7f4154100f70 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f41591b3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f415410cfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f41591b3700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f415410cfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39114/0 (socket says 172.21.15.90:39114) 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f41591b3700 1 -- 172.21.15.90:0/4036853549 learned_addr learned my addr 172.21.15.90:0/4036853549 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.214+0000 7f41589b2700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 0x7f415410d510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 -- 172.21.15.90:0/4036853549 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 msgr2=0x7f415410cfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f415410cfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 -- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f415419de60 msgr2=0x7f415419e2e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:20:31.217 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f415419de60 0x7f415419e2e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 -- 172.21.15.90:0/4036853549 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f414c009d00 con 0x7f4154106090 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f41589b2700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 0x7f415410d510 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f414400e9c0 tx=0x7f414400ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f414a7fc700 1 -- 172.21.15.90:0/4036853549 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f414400cbd0 con 0x7f4154106090 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f414a7fc700 1 -- 172.21.15.90:0/4036853549 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f414400eeb0 con 0x7f4154106090 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f414a7fc700 1 -- 172.21.15.90:0/4036853549 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4144018680 con 0x7f4154106090 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f41541a6170 con 0x7f4154106090 2024-04-06T14:20:31.218 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.215+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41541a6650 con 0x7f4154106090 2024-04-06T14:20:31.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.217+0000 7f414a7fc700 1 -- 172.21.15.90:0/4036853549 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4144010430 con 0x7f4154106090 2024-04-06T14:20:31.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.218+0000 7f414a7fc700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 0x7f41400a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.218+0000 7f414a7fc700 1 -- 172.21.15.90:0/4036853549 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4144014070 con 0x7f4154106090 2024-04-06T14:20:31.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.218+0000 7f41591b3700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 0x7f41400a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.219 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.218+0000 7f41591b3700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 0x7f41400a37d0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f4150009ad0 tx=0x7f4150009290 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.220 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 Processor -- start 2024-04-06T14:20:31.221 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 -- start start 2024-04-06T14:20:31.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac3584b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac358e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac34dc990 con 0x555ac35850a0 2024-04-06T14:20:31.222 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac34da350 con 0x555ac3584780 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.220+0000 7f41675cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac34d5c40 con 0x555ac358ede0 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39118/0 (socket says 172.21.15.90:39118) 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 -- 172.21.15.90:0/768169175 learned_addr learned my addr 172.21.15.90:0/768169175 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 -- 172.21.15.90:0/768169175 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 msgr2=0x555ac3584b60 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac3584b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 -- 172.21.15.90:0/768169175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 msgr2=0x555ac358e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac358e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 -- 172.21.15.90:0/768169175 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555ac33573b0 con 0x555ac358ede0 2024-04-06T14:20:31.223 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.221+0000 7f41599b4700 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f414c006130 tx=0x7f414c009c20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1a4c9554ca8e0633 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.222+0000 7f413f7fe700 1 -- 172.21.15.90:0/768169175 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f414c00e040 con 0x555ac358ede0 2024-04-06T14:20:31.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.222+0000 7f413f7fe700 1 -- 172.21.15.90:0/768169175 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f414c007cc0 con 0x555ac358ede0 2024-04-06T14:20:31.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.223+0000 7f413f7fe700 1 -- 172.21.15.90:0/768169175 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f414c007700 con 0x555ac358ede0 2024-04-06T14:20:31.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.223+0000 7f41675cbb80 1 -- 172.21.15.90:0/768169175 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 msgr2=0x555ac35911d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.226 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.223+0000 7f41675cbb80 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f414c006130 tx=0x7f414c009c20 comp rx=0 tx=0).stop 2024-04-06T14:20:31.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.245+0000 7f41675cbb80 1 -- 172.21.15.90:0/768169175 shutdown_connections 2024-04-06T14:20:31.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.245+0000 7f41675cbb80 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac35911d0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.245+0000 7f41675cbb80 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac3584b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.245+0000 7f41675cbb80 1 --2- 172.21.15.90:0/768169175 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac358e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.245+0000 7f41675cbb80 1 -- 172.21.15.90:0/768169175 >> 172.21.15.90:0/768169175 conn(0x555ac34e36a0 msgr2=0x555ac357fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.256 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.256+0000 7f41675cbb80 1 -- 172.21.15.90:0/768169175 shutdown_connections 2024-04-06T14:20:31.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.266+0000 7f41675cbb80 1 -- 172.21.15.90:0/768169175 wait complete. 2024-04-06T14:20:31.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.266+0000 7f41675cbb80 1 -- start start 2024-04-06T14:20:31.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.266+0000 7f41675cbb80 1 Processor -- start 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 0x555ac358bc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac358c160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 0x555ac358c6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac3471d80 con 0x555ac3584780 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac3471a20 con 0x555ac358ede0 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac33596a0 con 0x555ac35850a0 2024-04-06T14:20:31.268 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.267+0000 7f41675cbb80 10 client.0 _ll_register_callbacks cb 0x555ac347b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.268+0000 7f41599b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 0x555ac358c6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.268+0000 7f41599b4700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 0x555ac358c6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38030/0 (socket says 172.21.15.90:38030) 2024-04-06T14:20:31.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.268+0000 7f41599b4700 1 -- 172.21.15.90:0/377355749 learned_addr learned my addr 172.21.15.90:0/377355749 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.268+0000 7f41589b2700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac358c160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.269 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.269+0000 7f41591b3700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 0x555ac358bc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.270+0000 7f41591b3700 1 -- 172.21.15.90:0/377355749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 msgr2=0x555ac358c160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.270+0000 7f41591b3700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac358c160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.270 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.270+0000 7f41591b3700 1 -- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 msgr2=0x555ac358c6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.270+0000 7f41591b3700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 0x555ac358c6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.270+0000 7f41591b3700 1 -- 172.21.15.90:0/377355749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f415410e480 con 0x555ac3584780 2024-04-06T14:20:31.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.271+0000 7f41591b3700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 0x555ac358bc20 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7f415000e4b0 tx=0x7f4150004860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7cf54abda4289d58 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.271 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.271+0000 7f413cff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:20:31.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.275+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4150015bc0 con 0x555ac3584780 2024-04-06T14:20:31.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.275+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4150015d20 con 0x555ac3584780 2024-04-06T14:20:31.275 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.275+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555ac3496380 con 0x555ac3584780 2024-04-06T14:20:31.276 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.275+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4150005a50 con 0x555ac3584780 2024-04-06T14:20:31.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.277+0000 7f41675cbb80 10 client.17922 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.280 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.277+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555ac3303b60 con 0x555ac3584780 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f415002c030 con 0x555ac3584780 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4150013450 con 0x555ac3584780 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 10 client.17922 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 10 client.17922 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 20 client.17922 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac35b82b0 0x555ac35ba6d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 4 client.17922 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:31.282 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.281+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555ac34ce700 con 0x555ac35b82b0 2024-04-06T14:20:31.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.282+0000 7f41589b2700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac35b82b0 0x555ac35ba6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.283+0000 7f413cff9700 10 client.17922 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:20:31.283 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.283+0000 7f41589b2700 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac35b82b0 0x555ac35ba6d0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=cf660ceeac212dab server_cookie=659795a56f66a2cb in_seq=0 out_seq=0 2024-04-06T14:20:31.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.284+0000 7f413cff9700 1 -- 172.21.15.90:0/377355749 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555ac34ce700 con 0x555ac35b82b0 2024-04-06T14:20:31.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.284+0000 7f413cff9700 10 client.17922 handle_command_reply: tid=0 2024-04-06T14:20:31.284 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.284+0000 7f41675cbb80 1 client.17922 shutdown 2024-04-06T14:20:31.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 msgr2=0x555ac358bc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.287 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 0x555ac358bc20 secure :-1 s=READY pgs=871 cs=0 l=1 rev1=1 crypto rx=0x7f415000e4b0 tx=0x7f4150004860 comp rx=0 tx=0).stop 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 shutdown_connections 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac35b82b0 0x555ac35ba6d0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac358ede0 0x555ac358c6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac3584780 0x555ac358bc20 unknown :-1 s=CLOSED pgs=871 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 --2- 172.21.15.90:0/377355749 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac358c160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.287+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 >> 172.21.15.90:0/377355749 conn(0x555ac34e36a0 msgr2=0x555ac357ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.288 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.288+0000 7f413cff9700 0 client.17922 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:20:31.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:31 smithi090 ceph-mon[43420]: pgmap v474: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:20:31.291 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4166255419' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:20:31.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.292+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 shutdown_connections 2024-04-06T14:20:31.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.292+0000 7f41675cbb80 1 -- 172.21.15.90:0/377355749 wait complete. 2024-04-06T14:20:31.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.292+0000 7f41675cbb80 20 client.17922 trim_cache size 0 max 16384 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 Processor -- start 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 -- start start 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35911d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac3587480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac35b5a10 con 0x555ac35850a0 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac358e740 con 0x555ac3584780 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41675cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac3586290 con 0x555ac358ede0 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41589b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41589b2700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35294/0 (socket says 172.21.15.90:35294) 2024-04-06T14:20:31.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41589b2700 1 -- 172.21.15.90:0/3327929138 learned_addr learned my addr 172.21.15.90:0/3327929138 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41599b4700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac3587480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.311+0000 7f41591b3700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35911d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 -- 172.21.15.90:0/3327929138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 msgr2=0x555ac35911d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35911d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 -- 172.21.15.90:0/3327929138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 msgr2=0x555ac3586f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 -- 172.21.15.90:0/3327929138 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555ac349bea0 con 0x555ac358ede0 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41591b3700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35911d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41589b2700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f41599b4700 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac3587480 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f414c00b3f0 tx=0x7f414c007e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e2c2a2fa25acc500 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f413ffff700 1 -- 172.21.15.90:0/3327929138 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f414c00e040 con 0x555ac358ede0 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f413ffff700 1 -- 172.21.15.90:0/3327929138 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f414c0074a0 con 0x555ac358ede0 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.312+0000 7f413ffff700 1 -- 172.21.15.90:0/3327929138 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f414c024450 con 0x555ac358ede0 2024-04-06T14:20:31.318 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 -- 172.21.15.90:0/3327929138 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 msgr2=0x555ac3587480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac3587480 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f414c00b3f0 tx=0x7f414c007e30 comp rx=0 tx=0).stop 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 -- 172.21.15.90:0/3327929138 shutdown_connections 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac358ede0 0x555ac3587480 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35911d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3327929138 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac35850a0 0x555ac3586f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 -- 172.21.15.90:0/3327929138 >> 172.21.15.90:0/3327929138 conn(0x555ac34e36a0 msgr2=0x555ac358b160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 -- 172.21.15.90:0/3327929138 shutdown_connections 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.313+0000 7f41675cbb80 1 -- 172.21.15.90:0/3327929138 wait complete. 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 -- start start 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 Processor -- start 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac35ad280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.319 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac35adf10 con 0x555ac358ede0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac35ae080 con 0x555ac3584780 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555ac360b0e0 con 0x555ac35850a0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41675cbb80 10 client.0 _ll_register_callbacks cb 0x555ac347b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41599b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41591b3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41591b3700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38044/0 (socket says 172.21.15.90:38044) 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41599b4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35296/0 (socket says 172.21.15.90:35296) 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41591b3700 1 -- 172.21.15.90:0/3303491090 learned_addr learned my addr 172.21.15.90:0/3303491090 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.314+0000 7f41589b2700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac35ad280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 -- 172.21.15.90:0/3303491090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 msgr2=0x555ac35ad280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac35ad280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 -- 172.21.15.90:0/3303491090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 msgr2=0x555ac35ad800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.320 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 -- 172.21.15.90:0/3303491090 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x555ac3303b60 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41599b4700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41591b3700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f415000e4b0 tx=0x7f4150013f70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a2ae87f056194cc server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f413cff9700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4150015c90 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f414c007cc0 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4150015df0 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f415001f600 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41675cbb80 10 client.27674 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f41541a6650 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.315+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f414c007cc0 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f41500326a0 con 0x555ac3584780 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 10 client.27674 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 10 client.27674 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:20:31.321 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 20 client.27674 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac360c820 0x555ac360ec40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 4 client.27674 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x555ac3324cf0 con 0x555ac360c820 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41589b2700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac360c820 0x555ac360ec40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f413cff9700 10 client.27674 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41589b2700 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac360c820 0x555ac360ec40 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=691b313ccc0fe50b server_cookie=bde63388b0ecd095 in_seq=0 out_seq=0 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f413cff9700 1 -- 172.21.15.90:0/3303491090 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x555ac3324cf0 con 0x555ac360c820 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f413cff9700 10 client.27674 handle_command_reply: tid=0 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.316+0000 7f41675cbb80 1 client.27674 shutdown 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 msgr2=0x555ac35af490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f415000e4b0 tx=0x7f4150013f70 comp rx=0 tx=0).stop 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 shutdown_connections 2024-04-06T14:20:31.322 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x555ac360c820 0x555ac360ec40 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555ac358ede0 0x555ac35ad800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555ac3584780 0x555ac35af490 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 --2- 172.21.15.90:0/3303491090 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555ac35850a0 0x555ac35ad280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 >> 172.21.15.90:0/3303491090 conn(0x555ac34e36a0 msgr2=0x555ac358a1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f413cff9700 0 client.27674 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 shutdown_connections 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 1 -- 172.21.15.90:0/3303491090 wait complete. 2024-04-06T14:20:31.323 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:20:31.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f41675cbb80 20 client.27674 trim_cache size 0 max 16384 2024-04-06T14:20:31.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 msgr2=0x7f41400a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 0x7f41400a37d0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f4150009ad0 tx=0x7f4150009290 comp rx=0 tx=0).stop 2024-04-06T14:20:31.324 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 msgr2=0x7f415410d510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 0x7f415410d510 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7f414400e9c0 tx=0x7f414400ecd0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 shutdown_connections 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f41400a1310 0x7f41400a37d0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f415419de60 0x7f415419e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4154100f70 0x7f415410cfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.317+0000 7f415b417700 1 --2- 172.21.15.90:0/4036853549 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4154106090 0x7f415410d510 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.318+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 >> 172.21.15.90:0/4036853549 conn(0x7f41540fc7f0 msgr2=0x7f41540febe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.321+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 shutdown_connections 2024-04-06T14:20:31.325 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.321+0000 7f415b417700 1 -- 172.21.15.90:0/4036853549 wait complete. 2024-04-06T14:20:31.334 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:20:31.334 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:20:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:31 smithi105 ceph-mon[46969]: pgmap v474: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:20:31.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4166255419' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:20:31.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.544+0000 7fc7f28cc700 1 -- 172.21.15.90:0/1883246140 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 msgr2=0x7fc7ec10c280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.545 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.544+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/1883246140 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec10c280 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fc7e4009f90 tx=0x7fc7e400b330 comp rx=0 tx=0).stop 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 -- 172.21.15.90:0/1883246140 shutdown_connections 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/1883246140 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec10c280 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/1883246140 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec0ff5e0 0x7fc7ec0ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/1883246140 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7ec0fff90 0x7fc7ec103fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 -- 172.21.15.90:0/1883246140 >> 172.21.15.90:0/1883246140 conn(0x7fc7ec0761b0 msgr2=0x7fc7ec0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 -- 172.21.15.90:0/1883246140 shutdown_connections 2024-04-06T14:20:31.546 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.545+0000 7fc7f28cc700 1 -- 172.21.15.90:0/1883246140 wait complete. 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 Processor -- start 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 -- start start 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7ec0ff5e0 0x7fc7ec197360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec1978a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7ec19c670 con 0x7fc7ec0ff5e0 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7ec19c7e0 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.547 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.546+0000 7fc7f28cc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc7ec19c9a0 con 0x7fc7ec104700 2024-04-06T14:20:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7eb7fe700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec1978a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38062/0 (socket says 172.21.15.90:38062) 2024-04-06T14:20:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 -- 172.21.15.90:0/2406119475 learned_addr learned my addr 172.21.15.90:0/2406119475 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.548 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 -- 172.21.15.90:0/2406119475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 msgr2=0x7fc7ec1978a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec1978a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 -- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7ec0ff5e0 msgr2=0x7fc7ec197360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7ec0ff5e0 0x7fc7ec197360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.547+0000 7fc7f0e69700 1 -- 172.21.15.90:0/2406119475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7e4009c40 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.549 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.548+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fc7e4007c00 tx=0x7fc7e4007c30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.551+0000 7fc7e97fa700 1 -- 172.21.15.90:0/2406119475 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e400e040 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.551+0000 7fc7e97fa700 1 -- 172.21.15.90:0/2406119475 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7e4007e80 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.551+0000 7fc7e97fa700 1 -- 172.21.15.90:0/2406119475 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e401c660 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.551+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7ec10d8d0 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.551 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.551+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7ec10dd30 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.553+0000 7fc7e97fa700 1 -- 172.21.15.90:0/2406119475 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fc7e401c7c0 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.554 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.554+0000 7fc7e97fa700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 0x7fc7d40a38e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.555 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.554+0000 7fc7e97fa700 1 -- 172.21.15.90:0/2406119475 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc7e40ce680 con 0x7fc7ec19bbf0 2024-04-06T14:20:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.556+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 0x7fc7d40a38e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.556 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.556+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 0x7fc7d40a38e0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc005950 tx=0x7fc7dc0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.559 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 Processor -- start 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 -- start start 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c13b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c260b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993b6b990 con 0x55c993c13780 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993b69350 con 0x55c993c140a0 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993b64c40 con 0x55c993c265f0 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.561 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39174/0 (socket says 172.21.15.90:39174) 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7f0e69700 1 -- 172.21.15.90:0/140277773 learned_addr learned my addr 172.21.15.90:0/140277773 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.559+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c260b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c13b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 -- 172.21.15.90:0/140277773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 msgr2=0x55c993c260b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c260b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 -- 172.21.15.90:0/140277773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 msgr2=0x55c993c13b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c13b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 -- 172.21.15.90:0/140277773 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c9939e63b0 con 0x55c993c265f0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fc7e4098270 tx=0x7fc7e400b330 comp rx=0 tx=0).ready entity=mon.1 client_cookie=64ad2fb6397b9c1f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7d27fc700 1 -- 172.21.15.90:0/140277773 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e409b160 con 0x55c993c265f0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7d27fc700 1 -- 172.21.15.90:0/140277773 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7e409bf00 con 0x55c993c265f0 2024-04-06T14:20:31.562 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.560+0000 7fc7d27fc700 1 -- 172.21.15.90:0/140277773 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e40bb070 con 0x55c993c265f0 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140277773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 msgr2=0x55c993c289e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7fc7e4098270 tx=0x7fc7e400b330 comp rx=0 tx=0).stop 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140277773 shutdown_connections 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c265f0 0x55c993c289e0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c13b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140277773 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c260b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140277773 >> 172.21.15.90:0/140277773 conn(0x55c993b726a0 msgr2=0x55c993c0ea80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140277773 shutdown_connections 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140277773 wait complete. 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 -- start start 2024-04-06T14:20:31.567 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.561+0000 7fc7fea80b80 1 Processor -- start 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c1fe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c25610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993b00d80 con 0x55c993c13780 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993b00a20 con 0x55c993c265f0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9939e86a0 con 0x55c993c140a0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7fea80b80 10 client.0 _ll_register_callbacks cb 0x55c993b0a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c25610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35338/0 (socket says 172.21.15.90:35338) 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7f0e69700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c25610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38094/0 (socket says 172.21.15.90:38094) 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 -- 172.21.15.90:0/2284963058 learned_addr learned my addr 172.21.15.90:0/2284963058 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c1fe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.568 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 -- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 msgr2=0x55c993c1fe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c1fe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 -- 172.21.15.90:0/2284963058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 msgr2=0x55c993c25610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c25610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.562+0000 7fc7ebfff700 1 -- 172.21.15.90:0/2284963058 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7ec10dd30 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc01d6e0 tx=0x7fc7dc004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ea334f0e744d080f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7dc004a30 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc7dc004b90 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c993b25380 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc7dc004d50 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7fea80b80 10 client.17937 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c993992b60 con 0x55c993c13780 2024-04-06T14:20:31.569 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c993b25380 con 0x55c993c13780 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.563+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc7dc018070 con 0x55c993c13780 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 10 client.17937 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 10 client.17937 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 20 client.17937 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c3e850 0x55c993c40c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 4 client.17937 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c993b5d700 con 0x55c993c3e850 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.564+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c3e850 0x55c993c40c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7cb7fe700 10 client.17937 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c3e850 0x55c993c40c70 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=754304f630eb5348 server_cookie=df8a91b606a1f80b in_seq=0 out_seq=0 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/2284963058 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55c993b5d700 con 0x55c993c3e850 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7cb7fe700 10 client.17937 handle_command_reply: tid=0 2024-04-06T14:20:31.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7fea80b80 1 client.17937 shutdown 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 msgr2=0x55c993c1f8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.565+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 secure :-1 s=READY pgs=872 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc01d6e0 tx=0x7fc7dc004750 comp rx=0 tx=0).stop 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 shutdown_connections 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c3e850 0x55c993c40c70 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c25610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c1f8f0 unknown :-1 s=CLOSED pgs=872 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7cb7fe700 0 client.17937 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/2284963058 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c1fe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 >> 172.21.15.90:0/2284963058 conn(0x55c993b726a0 msgr2=0x55c993c0dc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 shutdown_connections 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 1 -- 172.21.15.90:0/2284963058 wait complete. 2024-04-06T14:20:31.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.566+0000 7fc7fea80b80 20 client.17937 trim_cache size 0 max 16384 2024-04-06T14:20:31.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 Processor -- start 2024-04-06T14:20:31.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 -- start start 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c13780 0x55c993c19c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 0x55c993c1fa50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c15ba0 con 0x55c993c265f0 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c15690 con 0x55c993c140a0 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c19240 con 0x55c993c13780 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7eb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7eb7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38106/0 (socket says 172.21.15.90:38106) 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7eb7fe700 1 -- 172.21.15.90:0/140367878 learned_addr learned my addr 172.21.15.90:0/140367878 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 0x55c993c1fa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.585+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c13780 0x55c993c19c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 -- 172.21.15.90:0/140367878 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c13780 msgr2=0x55c993c19c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c13780 0x55c993c19c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.587 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 -- 172.21.15.90:0/140367878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 msgr2=0x55c993c1fa50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 0x55c993c1fa50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 -- 172.21.15.90:0/140367878 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c993b2aea0 con 0x55c993c140a0 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 0x55c993c1fa50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fc7e0005950 tx=0x7fc7e000b410 comp rx=0 tx=0).ready entity=mon.2 client_cookie=221add2f1af3a2de server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7d2ffd700 1 -- 172.21.15.90:0/140367878 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e0016030 con 0x55c993c140a0 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7d2ffd700 1 -- 172.21.15.90:0/140367878 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7e0017040 con 0x55c993c140a0 2024-04-06T14:20:31.588 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.586+0000 7fc7d2ffd700 1 -- 172.21.15.90:0/140367878 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7e0004a90 con 0x55c993c140a0 2024-04-06T14:20:31.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140367878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 msgr2=0x55c993c1f510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7fc7e0005950 tx=0x7fc7e000b410 comp rx=0 tx=0).stop 2024-04-06T14:20:31.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140367878 shutdown_connections 2024-04-06T14:20:31.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c265f0 0x55c993c1fa50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.591 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c13780 0x55c993c19c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/140367878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c140a0 0x55c993c1f510 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140367878 >> 172.21.15.90:0/140367878 conn(0x55c993b726a0 msgr2=0x55c993c1aac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140367878 shutdown_connections 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- 172.21.15.90:0/140367878 wait complete. 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 -- start start 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.587+0000 7fc7fea80b80 1 Processor -- start 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c24800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c92690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c24f30 con 0x55c993c13780 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c92bd0 con 0x55c993c265f0 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c993c92d60 con 0x55c993c140a0 2024-04-06T14:20:31.592 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7fea80b80 10 client.0 _ll_register_callbacks cb 0x55c993b0a470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35358/0 (socket says 172.21.15.90:35358) 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 -- 172.21.15.90:0/4088801363 learned_addr learned my addr 172.21.15.90:0/4088801363 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c24800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7f0e69700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c92690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 -- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 msgr2=0x55c993c24800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c24800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 -- 172.21.15.90:0/4088801363 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 msgr2=0x55c993c92690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c92690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 -- 172.21.15.90:0/4088801363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7e000bc10 con 0x55c993c13780 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7ebfff700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc010c40 tx=0x7fc7dc010a10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1e05cb3f24ae9f4e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.588+0000 7fc7cb7fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:20:31.593 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7dc004830 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fc7dc004990 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc7e409bf00 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc7dc004b50 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 10 client.17943 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55c9939eacd0 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7dc045080 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fc7dc01d710 con 0x55c993c13780 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 10 client.17943 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 10 client.17943 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 20 client.17943 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c94510 0x55c993c96930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.594 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 4 client.17943 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.589+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c9939b3cf0 con 0x55c993c94510 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c94510 0x55c993c96930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7cb7fe700 10 client.17943 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7eb7fe700 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c94510 0x55c993c96930 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ac76d40528da968a server_cookie=7c86faa69149b141 in_seq=0 out_seq=0 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7cb7fe700 1 -- 172.21.15.90:0/4088801363 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c9939b3cf0 con 0x55c993c94510 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7cb7fe700 10 client.17943 handle_command_reply: tid=0 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 client.17943 shutdown 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 msgr2=0x55c993c242c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 secure :-1 s=READY pgs=873 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc010c40 tx=0x7fc7dc010a10 comp rx=0 tx=0).stop 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 shutdown_connections 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55c993c94510 0x55c993c96930 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7cb7fe700 0 client.17943 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.590+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55c993c265f0 0x55c993c92690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.595 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55c993c13780 0x55c993c242c0 unknown :-1 s=CLOSED pgs=873 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7fea80b80 1 --2- 172.21.15.90:0/4088801363 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55c993c140a0 0x55c993c24800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 >> 172.21.15.90:0/4088801363 conn(0x55c993b726a0 msgr2=0x55c993c12770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 shutdown_connections 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.591+0000 7fc7fea80b80 1 -- 172.21.15.90:0/4088801363 wait complete. 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7fea80b80 20 client.17943 trim_cache size 0 max 16384 2024-04-06T14:20:31.596 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:20:31.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 msgr2=0x7fc7d40a38e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 0x7fc7d40a38e0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fc7dc005950 tx=0x7fc7dc0058e0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.597 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 msgr2=0x7fc7ec19c070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7fc7e4007c00 tx=0x7fc7e4007c30 comp rx=0 tx=0).stop 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 shutdown_connections 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fc7d40a1420 0x7fc7d40a38e0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc7ec19bbf0 0x7fc7ec19c070 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fc7ec0ff5e0 0x7fc7ec197360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 --2- 172.21.15.90:0/2406119475 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fc7ec104700 0x7fc7ec1978a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 >> 172.21.15.90:0/2406119475 conn(0x7fc7ec0761b0 msgr2=0x7fc7ec102840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 shutdown_connections 2024-04-06T14:20:31.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.592+0000 7fc7f28cc700 1 -- 172.21.15.90:0/2406119475 wait complete. 2024-04-06T14:20:31.609 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:20:31.609 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:20:31.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:31 smithi134 ceph-mon[47549]: pgmap v474: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 16 MiB/s wr, 1.35k op/s 2024-04-06T14:20:31.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4166255419' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:20:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.858+0000 7f1991baf700 1 -- 172.21.15.90:0/197811821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c10f4b0 msgr2=0x7f198c107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.858 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.858+0000 7f1991baf700 1 --2- 172.21.15.90:0/197811821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c10f4b0 0x7f198c107db0 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f198400b0a0 tx=0x7f198400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 -- 172.21.15.90:0/197811821 shutdown_connections 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 --2- 172.21.15.90:0/197811821 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c11a590 0x7f198c11c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 --2- 172.21.15.90:0/197811821 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c10f4b0 0x7f198c107db0 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 --2- 172.21.15.90:0/197811821 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f198c10f920 0x7f198c1082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 -- 172.21.15.90:0/197811821 >> 172.21.15.90:0/197811821 conn(0x7f198c06be00 msgr2=0x7f198c06c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.859 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 -- 172.21.15.90:0/197811821 shutdown_connections 2024-04-06T14:20:31.861 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.859+0000 7f1991baf700 1 -- 172.21.15.90:0/197811821 wait complete. 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 Processor -- start 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 -- start start 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f198c10f4b0 0x7f198c118ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c11a590 0x7f198c111c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f198c119450 con 0x7f198c1121b0 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f198c116760 con 0x7f198c10f4b0 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.860+0000 7f1991baf700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f198c116920 con 0x7f198c11a590 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35366/0 (socket says 172.21.15.90:35366) 2024-04-06T14:20:31.862 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 -- 172.21.15.90:0/2543990778 learned_addr learned my addr 172.21.15.90:0/2543990778 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 -- 172.21.15.90:0/2543990778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c11a590 msgr2=0x7f198c111c70 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c11a590 0x7f198c111c70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 -- 172.21.15.90:0/2543990778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f198c10f4b0 msgr2=0x7f198c118ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f198c10f4b0 0x7f198c118ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 -- 172.21.15.90:0/2543990778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1984009d00 con 0x7f198c1121b0 2024-04-06T14:20:31.863 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.861+0000 7f198bfff700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f198000b450 tx=0x7f198000b810 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.862+0000 7f1988ff9700 1 -- 172.21.15.90:0/2543990778 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1980011830 con 0x7f198c1121b0 2024-04-06T14:20:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.862+0000 7f1988ff9700 1 -- 172.21.15.90:0/2543990778 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1980011e50 con 0x7f198c1121b0 2024-04-06T14:20:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.862+0000 7f1988ff9700 1 -- 172.21.15.90:0/2543990778 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f198000f2e0 con 0x7f198c1121b0 2024-04-06T14:20:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.862+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f198c116c00 con 0x7f198c1121b0 2024-04-06T14:20:31.864 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.862+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f198c1170c0 con 0x7f198c1121b0 2024-04-06T14:20:31.865 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.865+0000 7f1988ff9700 1 -- 172.21.15.90:0/2543990778 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f19800105a0 con 0x7f198c1121b0 2024-04-06T14:20:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.866+0000 7f1988ff9700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 0x7f19740a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.866+0000 7f1988ff9700 1 -- 172.21.15.90:0/2543990778 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f19800cc640 con 0x7f198c1121b0 2024-04-06T14:20:31.866 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.866+0000 7f198b7fe700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 0x7f19740a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.867 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.867+0000 7f198b7fe700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 0x7f19740a3810 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f198400b830 tx=0x7f1984006000 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.870 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.870+0000 7f199dd63b80 1 Processor -- start 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.870+0000 7f199dd63b80 1 -- start start 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92104db60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d9210578a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920fa5990 con 0x55d92104d780 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920fa3350 con 0x55d92104e0a0 2024-04-06T14:20:31.871 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.871+0000 7f199dd63b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920f9ec40 con 0x55d921057de0 2024-04-06T14:20:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.878+0000 7f198bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.878+0000 7f198affd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d9210578a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.878+0000 7f198bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:39238/0 (socket says 172.21.15.90:39238) 2024-04-06T14:20:31.878 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.878+0000 7f198bfff700 1 -- 172.21.15.90:0/860803712 learned_addr learned my addr 172.21.15.90:0/860803712 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198b7fe700 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92104db60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198bfff700 1 -- 172.21.15.90:0/860803712 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 msgr2=0x55d9210578a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198bfff700 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d9210578a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198bfff700 1 -- 172.21.15.90:0/860803712 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 msgr2=0x55d92104db60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198bfff700 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92104db60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.884+0000 7f198bfff700 1 -- 172.21.15.90:0/860803712 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d920e203b0 con 0x55d921057de0 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f198bfff700 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f1980095d40 tx=0x7f1980099490 comp rx=0 tx=0).ready entity=mon.1 client_cookie=87670b60dea00f1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f1971ffb700 1 -- 172.21.15.90:0/860803712 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f198009d670 con 0x55d921057de0 2024-04-06T14:20:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f1971ffb700 1 -- 172.21.15.90:0/860803712 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f19800a1cc0 con 0x55d921057de0 2024-04-06T14:20:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f1971ffb700 1 -- 172.21.15.90:0/860803712 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1980098d30 con 0x55d921057de0 2024-04-06T14:20:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f199dd63b80 1 -- 172.21.15.90:0/860803712 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 msgr2=0x55d92105a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.886 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.885+0000 7f199dd63b80 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f1980095d40 tx=0x7f1980099490 comp rx=0 tx=0).stop 2024-04-06T14:20:31.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.886+0000 7f199dd63b80 1 -- 172.21.15.90:0/860803712 shutdown_connections 2024-04-06T14:20:31.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.886+0000 7f199dd63b80 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d92105a1d0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.886+0000 7f199dd63b80 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92104db60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.886+0000 7f199dd63b80 1 --2- 172.21.15.90:0/860803712 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d9210578a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.886+0000 7f199dd63b80 1 -- 172.21.15.90:0/860803712 >> 172.21.15.90:0/860803712 conn(0x55d920fac6a0 msgr2=0x55d921048a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.889+0000 7f199dd63b80 1 -- 172.21.15.90:0/860803712 shutdown_connections 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.889+0000 7f199dd63b80 1 -- 172.21.15.90:0/860803712 wait complete. 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.889+0000 7f199dd63b80 1 -- start start 2024-04-06T14:20:31.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.889+0000 7f199dd63b80 1 Processor -- start 2024-04-06T14:20:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d921054c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 0x55d921055180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920f3ad80 con 0x55d92104d780 2024-04-06T14:20:31.891 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920f3aa20 con 0x55d921057de0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d920e226a0 con 0x55d92104e0a0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f199dd63b80 10 client.0 _ll_register_callbacks cb 0x55d920f44470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38174/0 (socket says 172.21.15.90:38174) 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 -- 172.21.15.90:0/3380066424 learned_addr learned my addr 172.21.15.90:0/3380066424 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198b7fe700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d921054c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198affd700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 0x55d921055180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 -- 172.21.15.90:0/3380066424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 msgr2=0x55d921055180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 0x55d921055180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 -- 172.21.15.90:0/3380066424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 msgr2=0x55d921054c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d921054c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198bfff700 1 -- 172.21.15.90:0/3380066424 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f198c1170c0 con 0x55d921057de0 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.890+0000 7f198affd700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 0x55d921055180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.892 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.891+0000 7f198bfff700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f198000bc10 tx=0x7f19800c9e60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=14a4d43f50422faf server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.891+0000 7f196affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:20:31.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1980098f30 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1980099aa0 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d920f5f380 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1980099c60 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f199dd63b80 10 client.27698 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.892+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d920dccb60 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d920f5f380 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f198009d170 con 0x55d921057de0 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 10 client.27698 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 10 client.27698 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 20 client.27698 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d921067650 0x55d921069a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 4 client.27698 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:31.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.893+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d920f97700 con 0x55d921067650 2024-04-06T14:20:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.895+0000 7f198b7fe700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d921067650 0x55d921069a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.895+0000 7f196affd700 10 client.27698 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:31.895 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.895+0000 7f198b7fe700 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d921067650 0x55d921069a40 crc :-1 s=READY pgs=252 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=527f8199982e5e17 server_cookie=de3bb13d48f728b9 in_seq=0 out_seq=0 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f196affd700 1 -- 172.21.15.90:0/3380066424 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d920f97700 con 0x55d921067650 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f196affd700 10 client.27698 handle_command_reply: tid=0 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 client.27698 shutdown 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 msgr2=0x55d9210556c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f198000bc10 tx=0x7f19800c9e60 comp rx=0 tx=0).stop 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 shutdown_connections 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d921057de0 0x55d9210556c0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d921054c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d921067650 0x55d921069a40 crc :-1 s=READY pgs=252 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 --2- 172.21.15.90:0/3380066424 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d92104e0a0 0x55d921055180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 >> 172.21.15.90:0/3380066424 conn(0x55d920fac6a0 msgr2=0x55d921047c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f196affd700 0 client.27698 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 shutdown_connections 2024-04-06T14:20:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.896+0000 7f199dd63b80 1 -- 172.21.15.90:0/3380066424 wait complete. 2024-04-06T14:20:31.898 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.897+0000 7f199dd63b80 20 client.27698 trim_cache size 0 max 16384 2024-04-06T14:20:31.915 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 Processor -- start 2024-04-06T14:20:31.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 -- start start 2024-04-06T14:20:31.921 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210483a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d92104f690 con 0x55d92104e0a0 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d9210561f0 con 0x55d92104d780 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f199dd63b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d92104ea40 con 0x55d921057de0 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35388/0 (socket says 172.21.15.90:35388) 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38184/0 (socket says 172.21.15.90:38184) 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198affd700 1 -- 172.21.15.90:0/2796649099 learned_addr learned my addr 172.21.15.90:0/2796649099 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.915+0000 7f198bfff700 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210483a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 -- 172.21.15.90:0/2796649099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 msgr2=0x55d9210483a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.922 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210483a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 -- 172.21.15.90:0/2796649099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 msgr2=0x55d921053510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 -- 172.21.15.90:0/2796649099 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d920f64ea0 con 0x55d92104e0a0 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198b7fe700 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f198affd700 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7f198c10fd40 tx=0x7f197c009f70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=10fbea9851de2ca8 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f19727fc700 1 -- 172.21.15.90:0/2796649099 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f197c0119a0 con 0x55d92104e0a0 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f19727fc700 1 -- 172.21.15.90:0/2796649099 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f197c014440 con 0x55d92104e0a0 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.916+0000 7f19727fc700 1 -- 172.21.15.90:0/2796649099 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f197c01d660 con 0x55d92104e0a0 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- 172.21.15.90:0/2796649099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 msgr2=0x55d921053a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.923 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 secure :-1 s=READY pgs=875 cs=0 l=1 rev1=1 crypto rx=0x7f198c10fd40 tx=0x7f197c009f70 comp rx=0 tx=0).stop 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- 172.21.15.90:0/2796649099 shutdown_connections 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210483a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104d780 0x55d921053510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 --2- 172.21.15.90:0/2796649099 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104e0a0 0x55d921053a50 unknown :-1 s=CLOSED pgs=875 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- 172.21.15.90:0/2796649099 >> 172.21.15.90:0/2796649099 conn(0x55d920fac6a0 msgr2=0x55d92104ff40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- 172.21.15.90:0/2796649099 shutdown_connections 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- 172.21.15.90:0/2796649099 wait complete. 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 -- start start 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.917+0000 7f199dd63b80 1 Processor -- start 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d92105c950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210ba410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d92105d0e0 con 0x55d92104d780 2024-04-06T14:20:31.924 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d92105d250 con 0x55d92104e0a0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d9210ba950 con 0x55d921057de0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f199dd63b80 10 client.0 _ll_register_callbacks cb 0x55d920f44470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35392/0 (socket says 172.21.15.90:35392) 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210ba410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198b7fe700 1 -- 172.21.15.90:0/1060757245 learned_addr learned my addr 172.21.15.90:0/1060757245 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198affd700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d92105c950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 -- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 msgr2=0x55d92105c950 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d92105c950 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 -- 172.21.15.90:0/1060757245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 msgr2=0x55d92105c410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 -- 172.21.15.90:0/1060757245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f198c11d270 con 0x55d921057de0 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198b7fe700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:31.925 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f198bfff700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210ba410 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f1980009fd0 tx=0x7f1980084520 comp rx=0 tx=0).ready entity=mon.1 client_cookie=31dff1dc0380340d server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.918+0000 7f196affd700 10 client.0 ms_handle_connect on v2:172.21.15.134:3300/0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f19800a1a30 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f19800a1cc0 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mon.1 v2:172.21.15.134:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f1980099f40 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f19800a6020 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 10 client.27433 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f197c0099c0 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mon.1 v2:172.21.15.134:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f19800a1cc0 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mon.1 v2:172.21.15.134:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f1980099bc0 con 0x55d921057de0 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 10 client.27433 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 10 client.27433 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 20 client.27433 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:31.926 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d9210bbc90 0x55d9210be0b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 4 client.27433 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.919+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d920dedcf0 con 0x55d9210bbc90 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f198b7fe700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d9210bbc90 0x55d9210be0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f198b7fe700 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d9210bbc90 0x55d9210be0b0 crc :-1 s=READY pgs=253 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f729a82dc31df6f6 server_cookie=ca2719bb717bc28c in_seq=0 out_seq=0 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f196affd700 10 client.27433 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f196affd700 1 -- 172.21.15.90:0/1060757245 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d920dedcf0 con 0x55d9210bbc90 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f196affd700 10 client.27433 handle_command_reply: tid=0 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 client.27433 shutdown 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 msgr2=0x55d9210ba410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210ba410 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f1980009fd0 tx=0x7f1980084520 comp rx=0 tx=0).stop 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 shutdown_connections 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55d9210bbc90 0x55d9210be0b0 crc :-1 s=READY pgs=253 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d921057de0 0x55d9210ba410 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.927 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f196affd700 0 client.27433 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d92104d780 0x55d92105c410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 --2- 172.21.15.90:0/1060757245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d92104e0a0 0x55d92105c950 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.920+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 >> 172.21.15.90:0/1060757245 conn(0x55d920fac6a0 msgr2=0x55d921053100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.921+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 shutdown_connections 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.921+0000 7f199dd63b80 1 -- 172.21.15.90:0/1060757245 wait complete. 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.921+0000 7f199dd63b80 20 client.27433 trim_cache size 0 max 16384 2024-04-06T14:20:31.928 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 msgr2=0x7f19740a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 0x7f19740a3810 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f198400b830 tx=0x7f1984006000 comp rx=0 tx=0).stop 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 msgr2=0x7f198c112660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 secure :-1 s=READY pgs=874 cs=0 l=1 rev1=1 crypto rx=0x7f198000b450 tx=0x7f198000b810 comp rx=0 tx=0).stop 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 shutdown_connections 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f19740a1350 0x7f19740a3810 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f198c1121b0 0x7f198c112660 unknown :-1 s=CLOSED pgs=874 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.929 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f198c10f4b0 0x7f198c118ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 --2- 172.21.15.90:0/2543990778 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f198c11a590 0x7f198c111c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:31.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.922+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 >> 172.21.15.90:0/2543990778 conn(0x7f198c06be00 msgr2=0x7f198c06f7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:31.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.923+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 shutdown_connections 2024-04-06T14:20:31.930 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:31.923+0000 7f1991baf700 1 -- 172.21.15.90:0/2543990778 wait complete. 2024-04-06T14:20:32.940 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2024-04-06T14:20:32.940 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:20:32.941 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 300 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2024-04-06T14:20:33.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 885.0 s, 1488.0 tps, lat 67.467 ms stddev 81.848 2024-04-06T14:20:33.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 -- 172.21.15.90:0/3630358547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 msgr2=0x7f3e5410cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.113 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/3630358547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e5410cb00 secure :-1 s=READY pgs=876 cs=0 l=1 rev1=1 crypto rx=0x7f3e4c00b0a0 tx=0x7f3e4c00b3b0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 -- 172.21.15.90:0/3630358547 shutdown_connections 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/3630358547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e5410cb00 unknown :-1 s=CLOSED pgs=876 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/3630358547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 0x7f3e54100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/3630358547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e54100820 0x7f3e54104850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 -- 172.21.15.90:0/3630358547 >> 172.21.15.90:0/3630358547 conn(0x7f3e54076d40 msgr2=0x7f3e54077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 -- 172.21.15.90:0/3630358547 shutdown_connections 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 -- 172.21.15.90:0/3630358547 wait complete. 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.111+0000 7f3e5ab21700 1 Processor -- start 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 -- start start 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 0x7f3e5419cfe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.114 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e54195ed0 con 0x7f3e54104f80 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e54196040 con 0x7f3e5419d520 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e5ab21700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e541b2a00 con 0x7f3e540ffe70 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38218/0 (socket says 172.21.15.90:38218) 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e53fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35408/0 (socket says 172.21.15.90:35408) 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 -- 172.21.15.90:0/2743988333 learned_addr learned my addr 172.21.15.90:0/2743988333 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 0x7f3e5419cfe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 -- 172.21.15.90:0/2743988333 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 msgr2=0x7f3e5419cfe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 0x7f3e5419cfe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 -- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 msgr2=0x7f3e541951c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 -- 172.21.15.90:0/2743988333 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e4c009d00 con 0x7f3e5419d520 2024-04-06T14:20:33.115 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e53fff700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e590be700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f3e4c00bdc0 tx=0x7f3e4c00bdf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.112+0000 7f3e51ffb700 1 -- 172.21.15.90:0/2743988333 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e4c00e040 con 0x7f3e5419d520 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.113+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e541b2c50 con 0x7f3e5419d520 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.113+0000 7f3e51ffb700 1 -- 172.21.15.90:0/2743988333 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e4c01d480 con 0x7f3e5419d520 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.113+0000 7f3e51ffb700 1 -- 172.21.15.90:0/2743988333 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e4c01b6b0 con 0x7f3e5419d520 2024-04-06T14:20:33.116 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.113+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e541b3120 con 0x7f3e5419d520 2024-04-06T14:20:33.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.115+0000 7f3e51ffb700 1 -- 172.21.15.90:0/2743988333 <== mon.2 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f3e4c01d8a0 con 0x7f3e5419d520 2024-04-06T14:20:33.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.115+0000 7f3e51ffb700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 0x7f3e440a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.119 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.116+0000 7f3e51ffb700 1 -- 172.21.15.90:0/2743988333 <== mon.2 v2:172.21.15.105:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3e4c0ce160 con 0x7f3e5419d520 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.116+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 0x7f3e440a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.116+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 0x7f3e440a3750 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f3e48009c60 tx=0x7f3e48009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 Processor -- start 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 -- start start 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 0x55ae27674be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27681250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae275cca10 con 0x55ae2767ee60 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae275ca3d0 con 0x55ae27675120 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae275c5cc0 con 0x55ae27674800 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e590be700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27681250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 0x55ae27674be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.120 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38232/0 (socket says 172.21.15.90:38232) 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 -- 172.21.15.90:0/3253511276 learned_addr learned my addr 172.21.15.90:0/3253511276 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 -- 172.21.15.90:0/3253511276 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 msgr2=0x55ae27674be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 0x55ae27674be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 -- 172.21.15.90:0/3253511276 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 msgr2=0x55ae27681250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27681250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e53fff700 1 -- 172.21.15.90:0/3253511276 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ae27447430 con 0x55ae27675120 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e590be700 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27681250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.117+0000 7f3e588bd700 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 0x55ae27674be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e53fff700 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f3e40011440 tx=0x7f3e40011750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e1779729deefa7e7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e3effd700 1 -- 172.21.15.90:0/3253511276 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e40004230 con 0x55ae27675120 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e3effd700 1 -- 172.21.15.90:0/3253511276 <== mon.2 v2:172.21.15.105:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e40004cf0 con 0x55ae27675120 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e3effd700 1 -- 172.21.15.90:0/3253511276 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e40005660 con 0x55ae27675120 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/3253511276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 msgr2=0x55ae2767e920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.121 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.118+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7f3e40011440 tx=0x7f3e40011750 comp rx=0 tx=0).stop 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/3253511276 shutdown_connections 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27681250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27674800 0x55ae27674be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/3253511276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27675120 0x55ae2767e920 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/3253511276 >> 172.21.15.90:0/3253511276 conn(0x55ae275d3720 msgr2=0x55ae2766fb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/3253511276 shutdown_connections 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/3253511276 wait complete. 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- start start 2024-04-06T14:20:33.122 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 Processor -- start 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2767e8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2767bc90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae2767c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27561e00 con 0x55ae27674800 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27561aa0 con 0x55ae2767ee60 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27449720 con 0x55ae27675120 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e66cd5b80 10 client.0 _ll_register_callbacks cb 0x55ae2756b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.119+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2767e8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2767e8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35424/0 (socket says 172.21.15.90:35424) 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e588bd700 1 -- 172.21.15.90:0/1455814441 learned_addr learned my addr 172.21.15.90:0/1455814441 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.124 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae2767c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e53fff700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2767bc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 -- 172.21.15.90:0/1455814441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 msgr2=0x55ae2767bc90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2767bc90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 -- 172.21.15.90:0/1455814441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 msgr2=0x55ae2767e8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2767e8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 -- 172.21.15.90:0/1455814441 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e541b3120 con 0x55ae2767ee60 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e590be700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae2767c3d0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f3e4c0cb890 tx=0x7f3e4c096ff0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=991b6061443e2947 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e33fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e4c09b9e0 con 0x55ae2767ee60 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3e4c09b2e0 con 0x55ae2767ee60 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.120+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ae27586400 con 0x55ae2767ee60 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3e4c09f110 con 0x55ae2767ee60 2024-04-06T14:20:33.125 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 10 client.27716 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x7f3e40009c90 con 0x55ae2767ee60 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e4c07f020 con 0x55ae2767ee60 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3e4c0b3030 con 0x55ae2767ee60 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 10 client.27716 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 10 client.27716 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 20 client.27716 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae2768e800 0x55ae27690c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 4 client.27716 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.121+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ae275be780 con 0x55ae2768e800 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.123+0000 7f3e588bd700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae2768e800 0x55ae27690c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.123+0000 7f3e588bd700 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae2768e800 0x55ae27690c20 crc :-1 s=READY pgs=254 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=67fced3e7c85427a server_cookie=51cd125d95ae8208 in_seq=0 out_seq=0 2024-04-06T14:20:33.126 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.123+0000 7f3e33fff700 10 client.27716 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e33fff700 1 -- 172.21.15.90:0/1455814441 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ae275be780 con 0x55ae2768e800 2024-04-06T14:20:33.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e33fff700 10 client.27716 handle_command_reply: tid=0 2024-04-06T14:20:33.127 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 client.27716 shutdown 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 msgr2=0x55ae2767c3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae2767c3d0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f3e4c0cb890 tx=0x7f3e4c096ff0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 shutdown_connections 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae2767c3d0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2767e8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae2768e800 0x55ae27690c20 crc :-1 s=READY pgs=254 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/1455814441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2767bc90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 >> 172.21.15.90:0/1455814441 conn(0x55ae275d3720 msgr2=0x55ae2766eca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e33fff700 0 client.27716 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 shutdown_connections 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.124+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/1455814441 wait complete. 2024-04-06T14:20:33.128 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.125+0000 7f3e66cd5b80 20 client.27716 trim_cache size 0 max 16384 2024-04-06T14:20:33.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 Processor -- start 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 -- start start 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2768aac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae27685ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27675a90 con 0x55ae27674800 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27670780 con 0x55ae2767ee60 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27686020 con 0x55ae27675120 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e53fff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2768aac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:35428/0 (socket says 172.21.15.90:35428) 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e588bd700 1 -- 172.21.15.90:0/364025710 learned_addr learned my addr 172.21.15.90:0/364025710 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.149+0000 7f3e590be700 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae27685ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.150+0000 7f3e588bd700 1 -- 172.21.15.90:0/364025710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 msgr2=0x55ae2768aac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.150+0000 7f3e588bd700 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2768aac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.150+0000 7f3e588bd700 1 -- 172.21.15.90:0/364025710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 msgr2=0x55ae27685ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.150+0000 7f3e588bd700 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae27685ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.150+0000 7f3e588bd700 1 -- 172.21.15.90:0/364025710 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ae27695850 con 0x55ae27674800 2024-04-06T14:20:33.151 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.151+0000 7f3e588bd700 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 secure :-1 s=READY pgs=877 cs=0 l=1 rev1=1 crypto rx=0x7f3e480097c0 tx=0x7f3e48014000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=21ed9a45e4a0ceb0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.151+0000 7f3e3f7fe700 1 -- 172.21.15.90:0/364025710 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e48022070 con 0x55ae27674800 2024-04-06T14:20:33.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.151+0000 7f3e3f7fe700 1 -- 172.21.15.90:0/364025710 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e48023040 con 0x55ae27674800 2024-04-06T14:20:33.152 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.151+0000 7f3e3f7fe700 1 -- 172.21.15.90:0/364025710 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e4801d670 con 0x55ae27674800 2024-04-06T14:20:33.153 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/364025710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 msgr2=0x55ae2766fc20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 secure :-1 s=READY pgs=877 cs=0 l=1 rev1=1 crypto rx=0x7f3e480097c0 tx=0x7f3e48014000 comp rx=0 tx=0).stop 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/364025710 shutdown_connections 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae2767ee60 0x55ae27685ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae27674800 0x55ae2766fc20 unknown :-1 s=CLOSED pgs=877 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/364025710 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae2768aac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/364025710 >> 172.21.15.90:0/364025710 conn(0x55ae275d3720 msgr2=0x55ae2767d3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/364025710 shutdown_connections 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/364025710 wait complete. 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 -- start start 2024-04-06T14:20:33.154 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.153+0000 7f3e66cd5b80 1 Processor -- start 2024-04-06T14:20:33.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae276dd540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27684e50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.157 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27685530 con 0x55ae2767ee60 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae276856a0 con 0x55ae27674800 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ae27685860 con 0x55ae27675120 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e66cd5b80 10 client.0 _ll_register_callbacks cb 0x55ae2756b4f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:38262/0 (socket says 172.21.15.90:38262) 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 -- 172.21.15.90:0/2964848332 learned_addr learned my addr 172.21.15.90:0/2964848332 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e590be700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27684e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e53fff700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae276dd540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 -- 172.21.15.90:0/2964848332 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 msgr2=0x55ae276dd540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae276dd540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 -- 172.21.15.90:0/2964848332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 msgr2=0x55ae27684e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.158 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27684e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e588bd700 1 -- 172.21.15.90:0/2964848332 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e48004080 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.154+0000 7f3e590be700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27684e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e588bd700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f3e48009440 tx=0x7f3e480057f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f2155aece7b759f6 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e48022040 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3e4802be00 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3e40004cf0 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3e4802b0b0 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 10 client.27722 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ae273f3be0 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e40004cf0 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f3e48035bf0 con 0x55ae27674800 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 10 client.27722 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:33.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 10 client.27722 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 20 client.27722 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae276e4e00 0x55ae276e7220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 4 client.27722 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.155+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55ae27414d70 con 0x55ae276e4e00 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.156+0000 7f3e53fff700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae276e4e00 0x55ae276e7220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.156+0000 7f3e53fff700 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae276e4e00 0x55ae276e7220 crc :-1 s=READY pgs=255 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1bd3e829fb0ef856 server_cookie=4f406c5f601589ff in_seq=0 out_seq=0 2024-04-06T14:20:33.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.156+0000 7f3e33fff700 10 client.27722 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.157+0000 7f3e33fff700 1 -- 172.21.15.90:0/2964848332 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55ae27414d70 con 0x55ae276e4e00 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.157+0000 7f3e33fff700 10 client.27722 handle_command_reply: tid=0 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.157+0000 7f3e66cd5b80 1 client.27722 shutdown 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.158+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 msgr2=0x55ae276dd000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.158+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7f3e48009440 tx=0x7f3e480057f0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 shutdown_connections 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ae2767ee60 0x55ae27684e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ae27674800 0x55ae276dd000 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x55ae276e4e00 0x55ae276e7220 crc :-1 s=READY pgs=255 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 --2- 172.21.15.90:0/2964848332 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ae27675120 0x55ae276dd540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e33fff700 0 client.27722 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.160+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 >> 172.21.15.90:0/2964848332 conn(0x55ae275d3720 msgr2=0x55ae2768a930 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 shutdown_connections 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e66cd5b80 1 -- 172.21.15.90:0/2964848332 wait complete. 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e66cd5b80 20 client.27722 trim_cache size 0 max 16384 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stdout: "return_code": 0, 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stdout: "scrub_tag": "f63c98d2-fdea-4eb4-98ad-9691fe944fac", 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": "asynchronous" 2024-04-06T14:20:33.163 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:20:33.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 msgr2=0x7f3e440a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 0x7f3e440a3750 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f3e48009c60 tx=0x7f3e48009260 comp rx=0 tx=0).stop 2024-04-06T14:20:33.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 msgr2=0x7f3e54195700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f3e4c00bdc0 tx=0x7f3e4c00bdf0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 shutdown_connections 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f3e440a1290 0x7f3e440a3750 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3e5419d520 0x7f3e54195700 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f3e540ffe70 0x7f3e5419cfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 --2- 172.21.15.90:0/2743988333 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f3e54104f80 0x7f3e541951c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 >> 172.21.15.90:0/2743988333 conn(0x7f3e54076d40 msgr2=0x7f3e540fe6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.161+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 shutdown_connections 2024-04-06T14:20:33.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.162+0000 7f3e5ab21700 1 -- 172.21.15.90:0/2743988333 wait complete. 2024-04-06T14:20:33.177 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:20:33.177 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:20:33.293 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:33 smithi090 ceph-mon[43420]: pgmap v475: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 1.42k op/s 2024-04-06T14:20:33.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 -- 172.21.15.90:0/2636554254 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c111bb0 msgr2=0x7f4b5c113fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/2636554254 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c113fa0 secure :-1 s=READY pgs=878 cs=0 l=1 rev1=1 crypto rx=0x7f4b54009f90 tx=0x7f4b5400b330 comp rx=0 tx=0).stop 2024-04-06T14:20:33.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 -- 172.21.15.90:0/2636554254 shutdown_connections 2024-04-06T14:20:33.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/2636554254 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c113fa0 unknown :-1 s=CLOSED pgs=878 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.361 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/2636554254 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c0756c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/2636554254 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c075c00 0x7f4b5c111670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 -- 172.21.15.90:0/2636554254 >> 172.21.15.90:0/2636554254 conn(0x7f4b5c0fe9c0 msgr2=0x7f4b5c100de0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 -- 172.21.15.90:0/2636554254 shutdown_connections 2024-04-06T14:20:33.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.360+0000 7f4b61c0f700 1 -- 172.21.15.90:0/2636554254 wait complete. 2024-04-06T14:20:33.362 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 Processor -- start 2024-04-06T14:20:33.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 -- start start 2024-04-06T14:20:33.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.363 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c075c00 0x7f4b5c111650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c10a3b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c10ab80 con 0x7f4b5c0752e0 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c10acf0 con 0x7f4b5c075c00 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.361+0000 7f4b61c0f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5c10aeb0 con 0x7f4b5c111bb0 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46188/0 (socket says 172.21.15.90:46188) 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/96043076 learned_addr learned my addr 172.21.15.90:0/96043076 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5affd700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c075c00 0x7f4b5c111650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c10a3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/96043076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c111bb0 msgr2=0x7f4b5c10a3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c10a3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/96043076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c075c00 msgr2=0x7f4b5c111650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.364 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c075c00 0x7f4b5c111650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/96043076 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b440095f0 con 0x7f4b5c0752e0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7f4b4400ec10 tx=0x7f4b4400ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b58ff9700 1 -- 172.21.15.90:0/96043076 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4400cc90 con 0x7f4b5c0752e0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b58ff9700 1 -- 172.21.15.90:0/96043076 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b440044e0 con 0x7f4b5c0752e0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b58ff9700 1 -- 172.21.15.90:0/96043076 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b44010430 con 0x7f4b5c0752e0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b54009c40 con 0x7f4b5c0752e0 2024-04-06T14:20:33.365 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.362+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b5c10ef90 con 0x7f4b5c0752e0 2024-04-06T14:20:33.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.365+0000 7f4b58ff9700 1 -- 172.21.15.90:0/96043076 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4b44003fb0 con 0x7f4b5c0752e0 2024-04-06T14:20:33.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.365+0000 7f4b58ff9700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 0x7f4b480a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.365+0000 7f4b5affd700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 0x7f4b480a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.365+0000 7f4b58ff9700 1 -- 172.21.15.90:0/96043076 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4b44014070 con 0x7f4b5c0752e0 2024-04-06T14:20:33.366 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.366+0000 7f4b5affd700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 0x7f4b480a3750 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4b5c10b8d0 tx=0x7f4b4c009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.366+0000 7f4b6ddc3b80 1 Processor -- start 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.366+0000 7f4b6ddc3b80 1 -- start start 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.366+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea262780 0x555cea262b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.366+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 0x555cea26c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea26f1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea1ba990 con 0x555cea2630a0 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea1b8350 con 0x555cea262780 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea1b3c40 con 0x555cea26cde0 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 0x555cea26c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea26f1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 0x555cea26c8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46198/0 (socket says 172.21.15.90:46198) 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5affd700 1 -- 172.21.15.90:0/3525982455 learned_addr learned my addr 172.21.15.90:0/3525982455 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.368 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 -- 172.21.15.90:0/3525982455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea262780 msgr2=0x555cea262b60 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea262780 0x555cea262b60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 -- 172.21.15.90:0/3525982455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 msgr2=0x555cea26c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 0x555cea26c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 -- 172.21.15.90:0/3525982455 --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555cea0353b0 con 0x555cea26cde0 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea26f1d0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f4b5400b6c0 tx=0x7f4b54009b70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4f4f8a14a0a7bf60 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b51ffb700 1 -- 172.21.15.90:0/3525982455 <== mon.1 v2:172.21.15.134:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b5400e040 con 0x555cea26cde0 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b51ffb700 1 -- 172.21.15.90:0/3525982455 <== mon.1 v2:172.21.15.134:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b540077a0 con 0x555cea26cde0 2024-04-06T14:20:33.369 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.367+0000 7f4b51ffb700 1 -- 172.21.15.90:0/3525982455 <== mon.1 v2:172.21.15.134:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b54007dd0 con 0x555cea26cde0 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3525982455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 msgr2=0x555cea26f1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea26f1d0 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f4b5400b6c0 tx=0x7f4b54009b70 comp rx=0 tx=0).stop 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3525982455 shutdown_connections 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea26f1d0 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea262780 0x555cea262b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3525982455 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea2630a0 0x555cea26c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.370 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.368+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3525982455 >> 172.21.15.90:0/3525982455 conn(0x555cea1c16a0 msgr2=0x555cea25da80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.372+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3525982455 shutdown_connections 2024-04-06T14:20:33.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.372+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3525982455 wait complete. 2024-04-06T14:20:33.372 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.372+0000 7f4b6ddc3b80 1 -- start start 2024-04-06T14:20:33.373 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.372+0000 7f4b6ddc3b80 1 Processor -- start 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea269c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 0x555cea289ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea14fd80 con 0x555cea26cde0 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea14fa20 con 0x555cea2630a0 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea0376a0 con 0x555cea26a710 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 10 client.0 _ll_register_callbacks cb 0x555cea159470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46206/0 (socket says 172.21.15.90:46206) 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 -- 172.21.15.90:0/3500891011 learned_addr learned my addr 172.21.15.90:0/3500891011 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.375 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 0x555cea289ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea269c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 -- 172.21.15.90:0/3500891011 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 msgr2=0x555cea289ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 0x555cea289ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 -- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 msgr2=0x555cea269c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea269c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 -- 172.21.15.90:0/3500891011 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b5c114a90 con 0x555cea26cde0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 0x555cea289ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea269c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b5affd700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 secure :-1 s=READY pgs=880 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c0154b0 tx=0x7f4b4c004860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=17efdb50288ff0fa server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b3affd700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c015cf0 con 0x555cea26cde0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4b4c013450 con 0x555cea26cde0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.373+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555cea174380 con 0x555cea26cde0 2024-04-06T14:20:33.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4b4c015e50 con 0x555cea26cde0 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 10 client.17988 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555ce9fe1b60 con 0x555cea26cde0 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555cea174380 con 0x555cea26cde0 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4b4c013d70 con 0x555cea26cde0 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 10 client.17988 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 10 client.17988 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 20 client.17988 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea28df20 0x555cea290340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 4 client.17988 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555cea1ac700 con 0x555cea28df20 2024-04-06T14:20:33.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.374+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea28df20 0x555cea290340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.375+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea28df20 0x555cea290340 crc :-1 s=READY pgs=256 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=657615e8c159ff06 server_cookie=fc01eb0753d69b4d in_seq=0 out_seq=0 2024-04-06T14:20:33.378 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.375+0000 7f4b3affd700 10 client.17988 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b3affd700 1 -- 172.21.15.90:0/3500891011 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x555cea1ac700 con 0x555cea28df20 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b3affd700 10 client.17988 handle_command_reply: tid=0 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 client.17988 shutdown 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 msgr2=0x555cea26a1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 secure :-1 s=READY pgs=880 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c0154b0 tx=0x7f4b4c004860 comp rx=0 tx=0).stop 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 shutdown_connections 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea28df20 0x555cea290340 crc :-1 s=READY pgs=256 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26a710 0x555cea289ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b3affd700 0 client.17988 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea269c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/3500891011 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26cde0 0x555cea26a1d0 unknown :-1 s=CLOSED pgs=880 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 >> 172.21.15.90:0/3500891011 conn(0x555cea1c16a0 msgr2=0x555cea25cc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 shutdown_connections 2024-04-06T14:20:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/3500891011 wait complete. 2024-04-06T14:20:33.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.376+0000 7f4b6ddc3b80 20 client.17988 trim_cache size 0 max 16384 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 Processor -- start 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 -- start start 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea25d520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea266f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea28aa90 con 0x555cea26a710 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea2807b0 con 0x555cea2630a0 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.400+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea26c800 con 0x555cea26cde0 2024-04-06T14:20:33.402 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:46220/0 (socket says 172.21.15.90:46220) 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 -- 172.21.15.90:0/149324528 learned_addr learned my addr 172.21.15.90:0/149324528 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea25d520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea266f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 -- 172.21.15.90:0/149324528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 msgr2=0x555cea266f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea266f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 -- 172.21.15.90:0/149324528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 msgr2=0x555cea25d520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea25d520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 -- 172.21.15.90:0/149324528 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555cea179ea0 con 0x555cea26a710 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.401+0000 7f4b5affd700 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c009fd0 tx=0x7f4b4c0062c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f94a116276523fd1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.402+0000 7f4b527fc700 1 -- 172.21.15.90:0/149324528 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c05c070 con 0x555cea26a710 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.402+0000 7f4b527fc700 1 -- 172.21.15.90:0/149324528 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b4c015870 con 0x555cea26a710 2024-04-06T14:20:33.403 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.402+0000 7f4b527fc700 1 -- 172.21.15.90:0/149324528 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4c021730 con 0x555cea26a710 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/149324528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 msgr2=0x555cea25da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 secure :-1 s=READY pgs=881 cs=0 l=1 rev1=1 crypto rx=0x7f4b4c009fd0 tx=0x7f4b4c0062c0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/149324528 shutdown_connections 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea266f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea25d520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/149324528 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea25da60 unknown :-1 s=CLOSED pgs=881 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/149324528 >> 172.21.15.90:0/149324528 conn(0x555cea1c16a0 msgr2=0x555cea268560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/149324528 shutdown_connections 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/149324528 wait complete. 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 -- start start 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 Processor -- start 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.408 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.403+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea2848a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b6ddc3b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea282a70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea282ff0 con 0x555cea26a710 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea283160 con 0x555cea2630a0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b6ddc3b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x555cea283320 con 0x555cea26cde0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b6ddc3b80 10 client.0 _ll_register_callbacks cb 0x555cea159470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.90:33682/0 (socket says 172.21.15.90:33682) 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/1148921028 learned_addr learned my addr 172.21.15.90:0/1148921028 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5bfff700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea282a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5affd700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea2848a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/1148921028 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 msgr2=0x555cea282a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea282a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.409 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/1148921028 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 msgr2=0x555cea2848a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea2848a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 -- 172.21.15.90:0/1148921028 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x555ce9fe1b60 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b5b7fe700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f4b4409a310 tx=0x7f4b440966c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=80eafbdb02d02e16 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.404+0000 7f4b3affd700 10 client.0 ms_handle_connect on v2:172.21.15.105:3300/0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mon.2 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4409aa80 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4b540077a0 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mon.2 v2:172.21.15.105:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4b4409a380 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mon.2 v2:172.21.15.105:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4b4409e1b0 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 10 client.27737 fetch_fsmap learned FSMap version 24 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x555cea039cd0 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mon.2 v2:172.21.15.105:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b540077a0 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mon.2 v2:172.21.15.105:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f4b4409a5e0 con 0x555cea2630a0 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 10 client.27737 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 10 client.27737 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:20:33.410 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 20 client.27737 populate_metadata read hostname 'smithi090' 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea27aa20 0x555cea27ce40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 4 client.27737 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.405+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x555cea002cf0 con 0x555cea27aa20 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b5affd700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea27aa20 0x555cea27ce40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b3affd700 10 client.27737 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b5affd700 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea27aa20 0x555cea27ce40 crc :-1 s=READY pgs=257 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=13b5b69f2efa7cd4 server_cookie=8c430957592ad536 in_seq=0 out_seq=0 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b3affd700 1 -- 172.21.15.90:0/1148921028 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+267 (crc 0 0 0) 0x555cea002cf0 con 0x555cea27aa20 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b3affd700 10 client.27737 handle_command_reply: tid=0 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 client.27737 shutdown 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 msgr2=0x555cea284360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f4b4409a310 tx=0x7f4b440966c0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 shutdown_connections 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x555cea27aa20 0x555cea27ce40 crc :-1 s=READY pgs=257 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.411 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x555cea26cde0 0x555cea282a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b3affd700 0 client.27737 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x555cea2630a0 0x555cea284360 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 --2- 172.21.15.90:0/1148921028 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x555cea26a710 0x555cea2848a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.406+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 >> 172.21.15.90:0/1148921028 conn(0x555cea1c16a0 msgr2=0x555cea26b530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.407+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 shutdown_connections 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.407+0000 7f4b6ddc3b80 1 -- 172.21.15.90:0/1148921028 wait complete. 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.407+0000 7f4b6ddc3b80 20 client.27737 trim_cache size 0 max 16384 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "scrub active (176 inodes in the stack)", 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": { 2024-04-06T14:20:33.412 INFO:teuthology.orchestra.run.smithi090.stdout: "f63c98d2-fdea-4eb4-98ad-9691fe944fac": { 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout: "path": "/", 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout: "tag": "f63c98d2-fdea-4eb4-98ad-9691fe944fac", 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout: "options": "recursive,force" 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:20:33.413 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 msgr2=0x7f4b480a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 0x7f4b480a3750 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4b5c10b8d0 tx=0x7f4b4c009190 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 msgr2=0x7f4b5c1110d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 secure :-1 s=READY pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7f4b4400ec10 tx=0x7f4b4400ef20 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 shutdown_connections 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4b480a1290 0x7f4b480a3750 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4b5c111bb0 0x7f4b5c10a3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4b5c0752e0 0x7f4b5c1110d0 secure :-1 s=CLOSED pgs=879 cs=0 l=1 rev1=1 crypto rx=0x7f4b4400ec10 tx=0x7f4b4400ef20 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 --2- 172.21.15.90:0/96043076 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b5c075c00 0x7f4b5c111650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 >> 172.21.15.90:0/96043076 conn(0x7f4b5c0fe9c0 msgr2=0x7f4b5c100db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 shutdown_connections 2024-04-06T14:20:33.414 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:33.408+0000 7f4b61c0f700 1 -- 172.21.15.90:0/96043076 wait complete. 2024-04-06T14:20:33.424 INFO:tasks.cephfs.filesystem:scrub status for tag:f63c98d2-fdea-4eb4-98ad-9691fe944fac - {'path': '/', 'tag': 'f63c98d2-fdea-4eb4-98ad-9691fe944fac', 'options': 'recursive,force'} 2024-04-06T14:20:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:33 smithi105 ceph-mon[46969]: pgmap v475: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 1.42k op/s 2024-04-06T14:20:33.651 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:33 smithi134 ceph-mon[47549]: pgmap v475: 97 pgs: 97 active+clean; 5.7 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 17 MiB/s wr, 1.42k op/s 2024-04-06T14:20:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:34 smithi105 ceph-mon[46969]: scrub queued for path: / 2024-04-06T14:20:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:34 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:34 smithi105 ceph-mon[46969]: scrub summary: active paths [/] 2024-04-06T14:20:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:34 smithi105 ceph-mon[46969]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:34 smithi090 ceph-mon[43420]: scrub queued for path: / 2024-04-06T14:20:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:34 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:34 smithi090 ceph-mon[43420]: scrub summary: active paths [/] 2024-04-06T14:20:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:34 smithi090 ceph-mon[43420]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:34.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:34 smithi134 ceph-mon[47549]: scrub queued for path: / 2024-04-06T14:20:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:34 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:34 smithi134 ceph-mon[47549]: scrub summary: active paths [/] 2024-04-06T14:20:34.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:34 smithi134 ceph-mon[47549]: scrub summary: idle+waiting paths [/] 2024-04-06T14:20:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:35 smithi105 ceph-mon[46969]: pgmap v476: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 16 MiB/s wr, 1.29k op/s 2024-04-06T14:20:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:35 smithi090 ceph-mon[43420]: pgmap v476: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 16 MiB/s wr, 1.29k op/s 2024-04-06T14:20:35.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:35 smithi134 ceph-mon[47549]: pgmap v476: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 16 MiB/s wr, 1.29k op/s 2024-04-06T14:20:36.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:36 smithi105 ceph-mon[46969]: scrub summary: idle 2024-04-06T14:20:36.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:36 smithi090 ceph-mon[43420]: scrub summary: idle 2024-04-06T14:20:36.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:36 smithi134 ceph-mon[47549]: scrub summary: idle 2024-04-06T14:20:37.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:37 smithi105 ceph-mon[46969]: pgmap v477: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 11 MiB/s wr, 793 op/s 2024-04-06T14:20:37.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:37 smithi090 ceph-mon[43420]: pgmap v477: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 11 MiB/s wr, 793 op/s 2024-04-06T14:20:37.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:37 smithi134 ceph-mon[47549]: pgmap v477: 97 pgs: 97 active+clean; 5.8 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 11 MiB/s wr, 793 op/s 2024-04-06T14:20:38.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 890.0 s, 1410.0 tps, lat 70.511 ms stddev 78.408 2024-04-06T14:20:39.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:39 smithi134 ceph-mon[47549]: pgmap v478: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 852 op/s 2024-04-06T14:20:39.501 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:39 smithi105 ceph-mon[46969]: pgmap v478: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 852 op/s 2024-04-06T14:20:39.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:39 smithi090 ceph-mon[43420]: pgmap v478: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 13 MiB/s wr, 852 op/s 2024-04-06T14:20:41.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:41 smithi105 ceph-mon[46969]: pgmap v479: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 9.9 MiB/s wr, 663 op/s 2024-04-06T14:20:41.543 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:41 smithi090 ceph-mon[43420]: pgmap v479: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 9.9 MiB/s wr, 663 op/s 2024-04-06T14:20:41.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:41 smithi134 ceph-mon[47549]: pgmap v479: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 9.9 MiB/s wr, 663 op/s 2024-04-06T14:20:43.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 895.0 s, 1547.2 tps, lat 64.415 ms stddev 79.377 2024-04-06T14:20:43.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:43 smithi105 ceph-mon[46969]: pgmap v480: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 10 MiB/s wr, 700 op/s 2024-04-06T14:20:43.542 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:43 smithi090 ceph-mon[43420]: pgmap v480: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 10 MiB/s wr, 700 op/s 2024-04-06T14:20:43.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:43 smithi134 ceph-mon[47549]: pgmap v480: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 10 MiB/s wr, 700 op/s 2024-04-06T14:20:44.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:44 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:20:44.542 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:44 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:20:44.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:44 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-06T14:20:45.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:45 smithi105 ceph-mon[46969]: pgmap v481: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 11 MiB/s wr, 736 op/s 2024-04-06T14:20:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:45 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:20:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:45 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:20:45.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:45 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:20:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:45 smithi090 ceph-mon[43420]: pgmap v481: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 11 MiB/s wr, 736 op/s 2024-04-06T14:20:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:45 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:20:45.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:45 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:20:45.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:45 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:20:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:45 smithi134 ceph-mon[47549]: pgmap v481: 97 pgs: 97 active+clean; 5.8 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 75 KiB/s rd, 11 MiB/s wr, 736 op/s 2024-04-06T14:20:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:45 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-06T14:20:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:45 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-06T14:20:45.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:45 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' 2024-04-06T14:20:47.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:47 smithi105 ceph-mon[46969]: pgmap v482: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 9.0 MiB/s wr, 582 op/s 2024-04-06T14:20:47.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:47 smithi090 ceph-mon[43420]: pgmap v482: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 9.0 MiB/s wr, 582 op/s 2024-04-06T14:20:47.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:47 smithi134 ceph-mon[47549]: pgmap v482: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 55 KiB/s rd, 9.0 MiB/s wr, 582 op/s 2024-04-06T14:20:48.023 INFO:teuthology.orchestra.run.smithi090.stderr:progress: 900.0 s, 1326.4 tps, lat 75.975 ms stddev 81.820 2024-04-06T14:20:48.255 INFO:teuthology.orchestra.run.smithi090.stdout:transaction type: 2024-04-06T14:20:48.255 INFO:teuthology.orchestra.run.smithi090.stdout:scaling factor: 32 2024-04-06T14:20:48.255 INFO:teuthology.orchestra.run.smithi090.stdout:query mode: simple 2024-04-06T14:20:48.255 INFO:teuthology.orchestra.run.smithi090.stdout:number of clients: 100 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:number of threads: 4 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:duration: 900 s 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:number of transactions actually processed: 1851832 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:latency average = 48.601 ms 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:latency stddev = 76.549 ms 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:tps = 2057.062445 (including connections establishing) 2024-04-06T14:20:48.256 INFO:teuthology.orchestra.run.smithi090.stdout:tps = 2057.067632 (excluding connections establishing) 2024-04-06T14:20:48.259 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo systemctl stop postgresql' 2024-04-06T14:20:49.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:49 smithi105 ceph-mon[46969]: pgmap v483: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 739 op/s 2024-04-06T14:20:49.544 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:49 smithi090 ceph-mon[43420]: pgmap v483: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 739 op/s 2024-04-06T14:20:49.646 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:49 smithi134 ceph-mon[47549]: pgmap v483: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 82 KiB/s rd, 11 MiB/s wr, 739 op/s 2024-04-06T14:20:51.541 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:51 smithi090 ceph-mon[43420]: pgmap v484: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 6.4 MiB/s wr, 512 op/s 2024-04-06T14:20:51.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:51 smithi134 ceph-mon[47549]: pgmap v484: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 6.4 MiB/s wr, 512 op/s 2024-04-06T14:20:51.748 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:51 smithi105 ceph-mon[46969]: pgmap v484: 97 pgs: 97 active+clean; 5.9 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 6.4 MiB/s wr, 512 op/s 2024-04-06T14:20:53.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:53 smithi134 ceph-mon[47549]: pgmap v485: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 8.7 MiB/s wr, 758 op/s 2024-04-06T14:20:53.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:53 smithi105 ceph-mon[46969]: pgmap v485: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 8.7 MiB/s wr, 758 op/s 2024-04-06T14:20:53.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:53 smithi090 ceph-mon[43420]: pgmap v485: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 8.7 MiB/s wr, 758 op/s 2024-04-06T14:20:55.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:55 smithi134 ceph-mon[47549]: pgmap v486: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-04-06T14:20:55.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:55 smithi105 ceph-mon[46969]: pgmap v486: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-04-06T14:20:55.788 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:55 smithi090 ceph-mon[43420]: pgmap v486: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 90 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-04-06T14:20:57.651 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:57 smithi090 ceph-mon[43420]: pgmap v487: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.8 MiB/s wr, 919 op/s 2024-04-06T14:20:57.660 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -lZaR /tmp/cephfs/postgres/' 2024-04-06T14:20:57.713 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/: 2024-04-06T14:20:57.713 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.713 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwxrwxrwt. 3 root root system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 data 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data: 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:total 31 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 . 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 .. 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 base 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 30 Apr 6 14:05 current_logfiles 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 59 Apr 6 14:05 global 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 log 2024-04-06T14:20:57.714 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_commit_ts 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_dynshmem 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 4269 Apr 6 14:05 pg_hba.conf 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 1636 Apr 6 14:05 pg_ident.conf 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 pg_logical 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 pg_multixact 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 pg_notify 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_replslot 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_serial 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_snapshots 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 pg_stat 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:20 pg_stat_tmp 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:20 pg_subtrans 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_tblspc 2024-04-06T14:20:57.715 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pg_twophase 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 64 Apr 6 14:20 pg_wal 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:12 pg_xact 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 88 Apr 6 14:05 postgresql.auto.conf 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 22989 Apr 6 14:05 postgresql.conf 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 51 Apr 6 14:05 postmaster.opts 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base: 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 . 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 1 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 12346 2024-04-06T14:20:57.716 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 308 Apr 6 14:05 12347 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 pgsql_tmp 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/1: 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:total 7233 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 . 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 .. 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:20:57.717 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:20:57.718 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:20:57.719 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:20:57.720 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:05 1259 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:20:57.721 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:20:57.722 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Apr 6 14:05 2608 2024-04-06T14:20:57.723 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:20:57.724 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Apr 6 14:05 2619 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2619_fsm 2024-04-06T14:20:57.725 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Apr 6 14:05 2658 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:20:57.726 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:20:57.727 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:20:57.728 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2696 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:20:57.729 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:20:57.730 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:20:57.731 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:20:57.732 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:20:57.733 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:20:57.734 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:20:57.735 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:20:57.736 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:20:57.736 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:20:57.736 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:20:57.738 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:20:57.738 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:20:57.739 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:20:57.740 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/12346: 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:total 7233 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Apr 6 14:05 . 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 .. 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:20:57.741 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:20:57.742 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:20:57.743 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:20:57.744 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:05 1259 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:20:57.745 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:20:57.746 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:20:57.747 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Apr 6 14:05 2608 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:20:57.748 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:20:57.749 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Apr 6 14:05 2619 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2619_fsm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:20:57.750 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Apr 6 14:05 2658 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:20:57.751 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:20:57.752 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2696 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:20:57.753 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:20:57.754 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:20:57.755 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:20:57.756 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:20:57.757 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:20:57.758 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:20:57.759 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:20:57.760 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:20:57.761 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:20:57.762 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:20:57.763 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/12347: 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:total 653616 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 308 Apr 6 14:05 . 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 .. 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 112 2024-04-06T14:20:57.764 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 113 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Apr 6 14:05 12182 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12182_fsm 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12182_vm 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12184 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12186 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12187_fsm 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12187_vm 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12189 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12191 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192 2024-04-06T14:20:57.765 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12192_fsm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12192_vm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12194 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12196 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12197_fsm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12197_vm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12199 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12201 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12202_fsm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12202_vm 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12204 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12206 2024-04-06T14:20:57.766 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 12207_fsm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12207_vm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12209 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12211 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12212 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 12214 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 12216 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:05 1247 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1247_fsm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1247_vm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Apr 6 14:05 1249 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1249_fsm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1249_vm 2024-04-06T14:20:57.767 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Apr 6 14:05 1255 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1255_fsm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1255_vm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:20 1259 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1259_fsm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1259_vm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1417_vm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 1418_vm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 11337728 Apr 6 14:20 16387 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:20 16387_fsm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:20 16387_vm 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 9445376 Apr 6 14:20 16393 2024-04-06T14:20:57.768 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:20 16393_fsm 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:20 16393_vm 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 469770240 Apr 6 14:20 16396 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 139264 Apr 6 14:20 16396_fsm 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:08 16396_vm 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Apr 6 14:20 16397 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 614400 Apr 6 14:20 16399 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 72736768 Apr 6 14:20 16401 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 97435648 Apr 6 14:20 16403 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:20 16403_fsm 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 174 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 175 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2187 2024-04-06T14:20:57.769 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2224_vm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2328_vm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2336_vm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2337 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2600 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2600_fsm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2600_vm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2601_fsm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2601_vm 2024-04-06T14:20:57.770 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2602 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2602_fsm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2602_vm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2603 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2603_fsm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2603_vm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2604_vm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2605 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2605_fsm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2605_vm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2606_fsm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2606_vm 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607 2024-04-06T14:20:57.771 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2607_fsm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2607_vm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 450560 Apr 6 14:05 2608 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2608_fsm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2608_vm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Apr 6 14:05 2609 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2609_fsm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2609_vm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2610 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2610_fsm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2610_vm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2611_vm 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612 2024-04-06T14:20:57.772 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2612_fsm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2612_vm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2613_vm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2615_fsm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2615_vm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2616_fsm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2616_vm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Apr 6 14:05 2617 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2617_fsm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2617_vm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Apr 6 14:05 2618 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2618_fsm 2024-04-06T14:20:57.773 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2618_vm 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 155648 Apr 6 14:20 2619 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:10 2619_fsm 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2619_vm 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2620_vm 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2650 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2651 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2652 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2653 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2654 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2655 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2656 2024-04-06T14:20:57.774 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2657 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 114688 Apr 6 14:05 2658 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2659 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2660 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2661 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2662 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2663 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2664 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2665 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2666 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2667 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2668 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2669 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2670 2024-04-06T14:20:57.775 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 352256 Apr 6 14:05 2673 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Apr 6 14:05 2674 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Apr 6 14:05 2675 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2678 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2679 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2680 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2681 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2682 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2683 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2684 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2685 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2686 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2687 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2688 2024-04-06T14:20:57.776 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Apr 6 14:05 2689 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Apr 6 14:05 2690 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Apr 6 14:05 2691 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2692 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2693 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:10 2696 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2699 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2701 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2702 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2703 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 2704 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2753 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2753_fsm 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2753_vm 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2754 2024-04-06T14:20:57.777 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2755 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2756 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 2757 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2830_vm 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2831 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2832_vm 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2833 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2834_vm 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2835 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836 2024-04-06T14:20:57.778 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2836_vm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2837 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Apr 6 14:05 2838 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2838_fsm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2838_vm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2839 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2840_fsm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2840_vm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2841 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2995_vm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2996 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3079_fsm 2024-04-06T14:20:57.779 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3079_vm 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3080 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3081 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3085 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3118_vm 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3119 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3164 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3256_vm 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3257 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3258 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3350_vm 2024-04-06T14:20:57.780 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3351 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3379 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3380 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3381_vm 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3394 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3394_fsm 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3394_vm 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3395 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3439_vm 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3440 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Apr 6 14:05 3455 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3456 2024-04-06T14:20:57.781 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3456_fsm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3456_vm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3466_vm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3467 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3468 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3501_vm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3502 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3503 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3534 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3541_fsm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3541_vm 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3542 2024-04-06T14:20:57.782 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3574 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3575 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3576_vm 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3596_vm 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3597 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3598_vm 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3599 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3600_fsm 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3600_vm 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601 2024-04-06T14:20:57.783 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3601_fsm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3601_vm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3602_fsm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3602_vm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3603 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3603_fsm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3603_vm 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3604 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3605 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3606 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3607 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3608 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Apr 6 14:05 3609 2024-04-06T14:20:57.784 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3712 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 3764_fsm 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3764_vm 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3766 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 3767 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3997 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 5002 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 548 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 549 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6102_vm 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6104_vm 2024-04-06T14:20:57.785 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6106_vm 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6110 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6111 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6112 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6113 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6117 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 826_vm 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 827 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 828 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 137860 Apr 6 14:05 pg_internal.init 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:05 PG_VERSION 2024-04-06T14:20:57.786 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/base/pgsql_tmp: 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 6 postgres postgres system_u:object_r:postgresql_db_t:s0 4 Apr 6 14:05 .. 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/global: 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:total 574 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 59 Apr 6 14:05 . 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1136 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1136_fsm 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1136_vm 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1137 2024-04-06T14:20:57.787 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1213 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1213_fsm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1213_vm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1214 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1214_fsm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1214_vm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1232 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 1233 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1260 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1260_fsm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1260_vm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1261 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1261_fsm 2024-04-06T14:20:57.788 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1261_vm 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1262 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 1262_fsm 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 1262_vm 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2396 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Apr 6 14:05 2396_fsm 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2396_vm 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2397 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2671 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2672 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2676 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2677 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2694 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2695 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2697 2024-04-06T14:20:57.789 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Apr 6 14:05 2698 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2846 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2846_vm 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2847 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2964 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2964_vm 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2965 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2966 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 2966_vm 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 2967 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3592 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 3592_vm 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 3593 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 4060 2024-04-06T14:20:57.790 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 4060_vm 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 4061 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6000 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6000_vm 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6001 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6002 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6100 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 6100_vm 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6114 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 6115 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:20 pg_control 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Apr 6 14:05 pg_filenode.map 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 21644 Apr 6 14:05 pg_internal.init 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/log: 2024-04-06T14:20:57.791 INFO:teuthology.orchestra.run.smithi090.stdout:total 3 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 2736 Apr 6 14:20 postgresql-Sat.log 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_commit_ts: 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_dynshmem: 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.792 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical: 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:total 1 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 . 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 mappings 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8 Apr 6 14:20 replorigin_checkpoint 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 snapshots 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical/mappings: 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 .. 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_logical/snapshots: 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.793 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.794 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:57 smithi105 ceph-mon[46969]: pgmap v487: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.8 MiB/s wr, 919 op/s 2024-04-06T14:20:57.795 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 .. 2024-04-06T14:20:57.795 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.795 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact: 2024-04-06T14:20:57.795 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 . 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 members 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 offsets 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact/members: 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 .. 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_multixact/offsets: 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:20:57.796 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:05 .. 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_notify: 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:total 8 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:05 . 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Apr 6 14:05 0000 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_replslot: 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.797 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_serial: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_snapshots: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_stat: 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:total 13 2024-04-06T14:20:57.798 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Apr 6 14:20 . 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 1864 Apr 6 14:20 db_0.stat 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 9300 Apr 6 14:20 db_12347.stat 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 607 Apr 6 14:20 global.stat 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_stat_tmp: 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:20 . 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_subtrans: 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:total 72 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Apr 6 14:20 . 2024-04-06T14:20:57.799 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Apr 6 14:20 001C 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_tblspc: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_twophase: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_wal: 2024-04-06T14:20:57.800 INFO:teuthology.orchestra.run.smithi090.stdout:total 1032192 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 64 Apr 6 14:20 . 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000000E 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000000F 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000010 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000011 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000012 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000013 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000014 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000015 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000016 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000017 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000018 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000019 2024-04-06T14:20:57.801 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001A 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001B 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001C 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001D 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001E 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 00000001000000020000001F 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000020 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000021 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000022 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000023 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000024 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:20 000000010000000200000025 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000026 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000027 2024-04-06T14:20:57.802 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000028 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000029 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 00000001000000020000002A 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000002B 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 00000001000000020000002C 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000002D 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 00000001000000020000002E 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000002F 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000030 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000031 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000032 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:17 000000010000000200000033 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000034 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000035 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000036 2024-04-06T14:20:57.803 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000037 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000038 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000039 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000003A 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000003B 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000003C 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000003D 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000003E 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000003F 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000040 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000041 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000042 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000043 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000044 2024-04-06T14:20:57.804 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000045 2024-04-06T14:20:57.805 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000046 2024-04-06T14:20:57.805 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 000000010000000200000047 2024-04-06T14:20:57.805 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000048 2024-04-06T14:20:57.805 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 000000010000000200000049 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000004A 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:19 00000001000000020000004B 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Apr 6 14:18 00000001000000020000004C 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 archive_status 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_wal/archive_status: 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:total 0 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Apr 6 14:05 . 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 64 Apr 6 14:20 .. 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:/tmp/cephfs/postgres/data/pg_xact: 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:total 456 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Apr 6 14:12 . 2024-04-06T14:20:57.806 INFO:teuthology.orchestra.run.smithi090.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 25 Apr 6 14:20 .. 2024-04-06T14:20:57.807 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 262144 Apr 6 14:12 0000 2024-04-06T14:20:57.807 INFO:teuthology.orchestra.run.smithi090.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 204800 Apr 6 14:20 0001 2024-04-06T14:20:57.807 DEBUG:teuthology.run_tasks:Unwinding manager full_sequential_finally 2024-04-06T14:20:57.819 INFO:teuthology.task.full_sequential_finally:In full_sequential_finally, running task exec... 2024-04-06T14:20:57.819 INFO:teuthology.task.exec:Executing custom commands... 2024-04-06T14:20:57.819 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:20:57.819 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > END_TIME' 2024-04-06T14:20:57.888 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi' 2024-04-06T14:20:57.892 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:57 smithi134 ceph-mon[47549]: pgmap v487: 97 pgs: 97 active+clean; 5.9 GiB data, 23 GiB used, 1.0 TiB / 1.0 TiB avail; 49 KiB/s rd, 9.8 MiB/s wr, 919 op/s 2024-04-06T14:20:57.937 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2024-04-06T14:20:58.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- 172.21.15.90:0/157346769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c106e30 msgr2=0x7efe9c107210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 --2- 172.21.15.90:0/157346769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c106e30 0x7efe9c107210 secure :-1 s=READY pgs=882 cs=0 l=1 rev1=1 crypto rx=0x7efe8c0099c0 tx=0x7efe8c009cd0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- 172.21.15.90:0/157346769 shutdown_connections 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 --2- 172.21.15.90:0/157346769 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 0x7efe9c10dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 --2- 172.21.15.90:0/157346769 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c106e30 0x7efe9c107210 unknown :-1 s=CLOSED pgs=882 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 --2- 172.21.15.90:0/157346769 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c0ff220 0x7efe9c0ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- 172.21.15.90:0/157346769 >> 172.21.15.90:0/157346769 conn(0x7efe9c0761b0 msgr2=0x7efe9c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- 172.21.15.90:0/157346769 shutdown_connections 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- 172.21.15.90:0/157346769 wait complete. 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 Processor -- start 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.072+0000 7efea3a40700 1 -- start start 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 0x7efe9c1976c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 0x7efe9c19ba60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe9c19c0a0 con 0x7efe9c0ff220 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe9c19c210 con 0x7efe9c106e30 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe9c19c3d0 con 0x7efe9c0ffbe0 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52026/0 (socket says 172.21.15.90:52026) 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 -- 172.21.15.90:0/566341431 learned_addr learned my addr 172.21.15.90:0/566341431 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea0fdb700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 0x7efe9c1976c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea1fdd700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 0x7efe9c19ba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 -- 172.21.15.90:0/566341431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 msgr2=0x7efe9c1976c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 0x7efe9c1976c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 -- 172.21.15.90:0/566341431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 msgr2=0x7efe9c19ba60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 0x7efe9c19ba60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 -- 172.21.15.90:0/566341431 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe8c0095f0 con 0x7efe9c0ff220 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea1fdd700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 0x7efe9c19ba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea17dc700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 secure :-1 s=READY pgs=883 cs=0 l=1 rev1=1 crypto rx=0x7efe8c0098e0 tx=0x7efe8c005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe8c01d070 con 0x7efe9c0ff220 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efe8c00fc30 con 0x7efe9c0ff220 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe9c19c650 con 0x7efe9c0ff220 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.073+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efe8c022e40 con 0x7efe9c0ff220 2024-04-06T14:20:58.076 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.074+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe9c1a3ef0 con 0x7efe9c0ff220 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.074+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efe9c109320 con 0x7efe9c0ff220 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.075+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7efe8c022570 con 0x7efe9c0ff220 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.076+0000 7efe927fc700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 0x7efe880a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.076+0000 7efea0fdb700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 0x7efe880a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.076+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7efe8c0ce6e0 con 0x7efe9c0ff220 2024-04-06T14:20:58.077 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.076+0000 7efea0fdb700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 0x7efe880a37d0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7efe9c1986d0 tx=0x7efe9800a380 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:58.079 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.080+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7efe8c097b40 con 0x7efe9c0ff220 2024-04-06T14:20:58.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.300+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) v1 -- 0x7efe9c1025e0 con 0x7efe880a1310 2024-04-06T14:20:58.307 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.308+0000 7efe927fc700 1 -- 172.21.15.90:0/566341431 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+350 (secure 0 0 0) 0x7efe9c1025e0 con 0x7efe880a1310 2024-04-06T14:20:58.307 INFO:teuthology.orchestra.run.smithi090.stdout:{"fs": "cephfs", "subvol": null, "group": null, "path": "/", "rel_path": "/", "schedule": "1m", "retention": {"m": 6, "h": 3}, "start": "2024-04-06T00:00:00", "created": "2024-04-06T14:05:32", "first": "2024-04-06T14:06:00", "last": "2024-04-06T14:20:00", "last_pruned": "2024-04-06T14:20:00", "created_count": 15, "pruned_count": 9, "active": true} 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 msgr2=0x7efe880a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 0x7efe880a37d0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7efe9c1986d0 tx=0x7efe9800a380 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 msgr2=0x7efe9c197180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 secure :-1 s=READY pgs=883 cs=0 l=1 rev1=1 crypto rx=0x7efe8c0098e0 tx=0x7efe8c005740 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 shutdown_connections 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7efe880a1310 0x7efe880a37d0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7efe9c106e30 0x7efe9c19ba60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7efe9c0ff220 0x7efe9c197180 unknown :-1 s=CLOSED pgs=883 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 --2- 172.21.15.90:0/566341431 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7efe9c0ffbe0 0x7efe9c1976c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 >> 172.21.15.90:0/566341431 conn(0x7efe9c0761b0 msgr2=0x7efe9c103b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:58.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 shutdown_connections 2024-04-06T14:20:58.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.310+0000 7efea3a40700 1 -- 172.21.15.90:0/566341431 wait complete. 2024-04-06T14:20:58.322 DEBUG:teuthology.orchestra.run.smithi090:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 -- 172.21.15.90:0/880154169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac0ff860 msgr2=0x7f4dac0ffc40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 --2- 172.21.15.90:0/880154169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac0ff860 0x7f4dac0ffc40 secure :-1 s=READY pgs=884 cs=0 l=1 rev1=1 crypto rx=0x7f4d9c009990 tx=0x7f4d9c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 -- 172.21.15.90:0/880154169 shutdown_connections 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 --2- 172.21.15.90:0/880154169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac109470 0x7f4dac10fdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 --2- 172.21.15.90:0/880154169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac0ff860 0x7f4dac0ffc40 unknown :-1 s=CLOSED pgs=884 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 --2- 172.21.15.90:0/880154169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac100210 0x7f4dac108d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.449 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 -- 172.21.15.90:0/880154169 >> 172.21.15.90:0/880154169 conn(0x7f4dac074f40 msgr2=0x7f4dac075350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 -- 172.21.15.90:0/880154169 shutdown_connections 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.447+0000 7f4db16fc700 1 -- 172.21.15.90:0/880154169 wait complete. 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 Processor -- start 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 -- start start 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac0ff860 0x7f4dac1993c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac19dca0 0x7f4dac19e120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dac19e720 con 0x7f4dac109470 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dac19e890 con 0x7f4dac19dca0 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4db16fc700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dac19ea50 con 0x7f4dac0ff860 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4daa7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4daa7fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:52052/0 (socket says 172.21.15.90:52052) 2024-04-06T14:20:58.450 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.448+0000 7f4daa7fc700 1 -- 172.21.15.90:0/4015949516 learned_addr learned my addr 172.21.15.90:0/4015949516 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daaffd700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac0ff860 0x7f4dac1993c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4dab7fe700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac19dca0 0x7f4dac19e120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 -- 172.21.15.90:0/4015949516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac0ff860 msgr2=0x7f4dac1993c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac0ff860 0x7f4dac1993c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 -- 172.21.15.90:0/4015949516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac19dca0 msgr2=0x7f4dac19e120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac19dca0 0x7f4dac19e120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 -- 172.21.15.90:0/4015949516 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4da0009520 con 0x7f4dac109470 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4daa7fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 secure :-1 s=READY pgs=885 cs=0 l=1 rev1=1 crypto rx=0x7f4da000ea30 tx=0x7f4da000ed40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4da000cbd0 con 0x7f4dac109470 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4da000eed0 con 0x7f4dac109470 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d9c0095f0 con 0x7f4dac109470 2024-04-06T14:20:58.451 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4da0003d20 con 0x7f4dac109470 2024-04-06T14:20:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.449+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4dac1a62a0 con 0x7f4dac109470 2024-04-06T14:20:58.452 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.450+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4dac10b4f0 con 0x7f4dac109470 2024-04-06T14:20:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.451+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f4da0003ee0 con 0x7f4dac109470 2024-04-06T14:20:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.451+0000 7f4d93fff700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 0x7f4d940a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:20:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.452+0000 7f4daaffd700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 0x7f4d940a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:20:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.452+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f4da0014070 con 0x7f4dac109470 2024-04-06T14:20:58.453 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.452+0000 7f4daaffd700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 0x7f4d940a3750 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f4d9c00b2e0 tx=0x7f4d9c005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:20:58.455 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.456+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f4da0096460 con 0x7f4dac109470 2024-04-06T14:20:58.679 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.680+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 --> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f4dac19a730 con 0x7f4d940a1290 2024-04-06T14:20:58.686 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.687+0000 7f4d93fff700 1 -- 172.21.15.90:0/4015949516 <== mgr.15288 v2:172.21.15.90:6800/2273339784 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+10 (secure 0 0 0) 0x7f4dac19a730 con 0x7f4d940a1290 2024-04-06T14:20:58.687 INFO:teuthology.orchestra.run.smithi090.stdout:/ 1m 6m3h 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 msgr2=0x7f4d940a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 0x7f4d940a3750 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f4d9c00b2e0 tx=0x7f4d9c005fd0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 msgr2=0x7f4dac199900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 secure :-1 s=READY pgs=885 cs=0 l=1 rev1=1 crypto rx=0x7f4da000ea30 tx=0x7f4da000ed40 comp rx=0 tx=0).stop 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 shutdown_connections 2024-04-06T14:20:58.689 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f4d940a1290 0x7f4d940a3750 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4dac19dca0 0x7f4dac19e120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f4dac0ff860 0x7f4dac1993c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 --2- 172.21.15.90:0/4015949516 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f4dac109470 0x7f4dac199900 unknown :-1 s=CLOSED pgs=885 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.689+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 >> 172.21.15.90:0/4015949516 conn(0x7f4dac074f40 msgr2=0x7f4dac107000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.690+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 shutdown_connections 2024-04-06T14:20:58.690 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:20:58.690+0000 7f4db16fc700 1 -- 172.21.15.90:0/4015949516 wait complete. 2024-04-06T14:20:58.701 DEBUG:teuthology.run_tasks:Unwinding manager fwd_scrub 2024-04-06T14:20:58.711 INFO:tasks.fwd_scrub:joining ForwardScrubbers 2024-04-06T14:20:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:59 smithi134 ceph-mon[47549]: pgmap v488: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 16 MiB/s wr, 1.54k op/s 2024-04-06T14:20:59.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:20:59 smithi134 ceph-mon[47549]: from='client.18000 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:20:59.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:59 smithi105 ceph-mon[46969]: pgmap v488: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 16 MiB/s wr, 1.54k op/s 2024-04-06T14:20:59.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:20:59 smithi105 ceph-mon[46969]: from='client.18000 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:20:59.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:59 smithi090 ceph-mon[43420]: pgmap v488: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 16 MiB/s wr, 1.54k op/s 2024-04-06T14:20:59.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:20:59 smithi090 ceph-mon[43420]: from='client.18000 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:21:00.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:00 smithi134 ceph-mon[47549]: from='client.18006 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:21:00.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:00 smithi105 ceph-mon[46969]: from='client.18006 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:21:00.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:00 smithi090 ceph-mon[43420]: from='client.18006 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-06T14:21:01.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:01 smithi134 ceph-mon[47549]: pgmap v489: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 381 KiB/s rd, 17 MiB/s wr, 1.64k op/s 2024-04-06T14:21:01.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:01 smithi105 ceph-mon[46969]: pgmap v489: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 381 KiB/s rd, 17 MiB/s wr, 1.64k op/s 2024-04-06T14:21:01.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:01 smithi090 ceph-mon[43420]: pgmap v489: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 381 KiB/s rd, 17 MiB/s wr, 1.64k op/s 2024-04-06T14:21:03.425 INFO:teuthology.orchestra.run:Running command with timeout 900 2024-04-06T14:21:03.425 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 900 ceph --cluster ceph tell mds.1:0 scrub status 2024-04-06T14:21:03.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:03 smithi105 ceph-mon[46969]: pgmap v490: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 393 KiB/s rd, 19 MiB/s wr, 1.85k op/s 2024-04-06T14:21:03.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:03 smithi090 ceph-mon[43420]: pgmap v490: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 393 KiB/s rd, 19 MiB/s wr, 1.85k op/s 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3114625743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe0181060b0 msgr2=0x7fe01810dc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3114625743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe0181060b0 0x7fe01810dc30 secure :-1 s=READY pgs=886 cs=0 l=1 rev1=1 crypto rx=0x7fe01000b0a0 tx=0x7fe01000b3b0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3114625743 shutdown_connections 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3114625743 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe0181060b0 0x7fe01810dc30 unknown :-1 s=CLOSED pgs=886 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3114625743 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe018100f90 0x7fe018101370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3114625743 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018101940 0x7fe018105980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3114625743 >> 172.21.15.90:0/3114625743 conn(0x7fe0180fc7f0 msgr2=0x7fe0180fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3114625743 shutdown_connections 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.568+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3114625743 wait complete. 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 Processor -- start 2024-04-06T14:21:03.570 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 -- start start 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018100f90 0x7fe018197360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe0181060b0 0x7fe0181978a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe01819c670 con 0x7fe01819bbf0 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe01819c7e0 con 0x7fe0181060b0 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe01d5d0700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe01819c9a0 con 0x7fe018100f90 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34302/0 (socket says 172.21.15.90:34302) 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 -- 172.21.15.90:0/3270247850 learned_addr learned my addr 172.21.15.90:0/3270247850 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe0181060b0 0x7fe0181978a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018100f90 0x7fe018197360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 -- 172.21.15.90:0/3270247850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018100f90 msgr2=0x7fe018197360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018100f90 0x7fe018197360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.571 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 -- 172.21.15.90:0/3270247850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe0181060b0 msgr2=0x7fe0181978a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe0181060b0 0x7fe0181978a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 -- 172.21.15.90:0/3270247850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe010009d00 con 0x7fe01819bbf0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.569+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7fe010006130 tx=0x7fe010009300 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.570+0000 7fdffffff700 1 -- 172.21.15.90:0/3270247850 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe01000e040 con 0x7fe01819bbf0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.570+0000 7fdffffff700 1 -- 172.21.15.90:0/3270247850 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe010004e40 con 0x7fe01819bbf0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.570+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe0181a3ea0 con 0x7fe01819bbf0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.570+0000 7fdffffff700 1 -- 172.21.15.90:0/3270247850 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe01001b6b0 con 0x7fe01819bbf0 2024-04-06T14:21:03.572 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.570+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe0181a4320 con 0x7fe01819bbf0 2024-04-06T14:21:03.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.572+0000 7fdffffff700 1 -- 172.21.15.90:0/3270247850 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fe01001d480 con 0x7fe01819bbf0 2024-04-06T14:21:03.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.572+0000 7fdffffff700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 0x7fe0000a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.572+0000 7fdffffff700 1 -- 172.21.15.90:0/3270247850 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe0100ce5c0 con 0x7fe01819bbf0 2024-04-06T14:21:03.579 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.572+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 0x7fe0000a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 0x7fe0000a3750 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fe00c005950 tx=0x7fe00c00b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 Processor -- start 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 -- start start 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefdcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdefe1f0 0x560bbdf005e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.573+0000 7fe029784b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbde47990 con 0x560bbdeef780 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe029784b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbde45350 con 0x560bbdef00a0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe029784b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbde40c40 con 0x560bbdefe1f0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34310/0 (socket says 172.21.15.90:34310) 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 -- 172.21.15.90:0/2742443834 learned_addr learned my addr 172.21.15.90:0/2742443834 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe0167fc700 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefdcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.580 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe0177fe700 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdefe1f0 0x560bbdf005e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 -- 172.21.15.90:0/2742443834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdefe1f0 msgr2=0x560bbdf005e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdefe1f0 0x560bbdf005e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 -- 172.21.15.90:0/2742443834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 msgr2=0x560bbdefdcb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefdcb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 -- 172.21.15.90:0/2742443834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560bbdcc23b0 con 0x560bbdeef780 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fe016ffd700 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 secure :-1 s=READY pgs=888 cs=0 l=1 rev1=1 crypto rx=0x7fe00c010fa0 tx=0x7fe00c013000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=de5aa43c2841e38f server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fdffd7fa700 1 -- 172.21.15.90:0/2742443834 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe00c011db0 con 0x560bbdeef780 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fdffd7fa700 1 -- 172.21.15.90:0/2742443834 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe00c015070 con 0x560bbdeef780 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.574+0000 7fdffd7fa700 1 -- 172.21.15.90:0/2742443834 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe00c01c610 con 0x560bbdeef780 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- 172.21.15.90:0/2742443834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 msgr2=0x560bbdeefb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 secure :-1 s=READY pgs=888 cs=0 l=1 rev1=1 crypto rx=0x7fe00c010fa0 tx=0x7fe00c013000 comp rx=0 tx=0).stop 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- 172.21.15.90:0/2742443834 shutdown_connections 2024-04-06T14:21:03.581 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdefe1f0 0x560bbdf005e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdeefb60 unknown :-1 s=CLOSED pgs=888 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 --2- 172.21.15.90:0/2742443834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefdcb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- 172.21.15.90:0/2742443834 >> 172.21.15.90:0/2742443834 conn(0x560bbde4e6a0 msgr2=0x560bbdeeaa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- 172.21.15.90:0/2742443834 shutdown_connections 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- 172.21.15.90:0/2742443834 wait complete. 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.575+0000 7fe029784b80 1 -- start start 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 Processor -- start 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdefdba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefaf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdddcd80 con 0x560bbdefe1f0 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdddca20 con 0x560bbdef00a0 2024-04-06T14:21:03.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdcc46a0 con 0x560bbdeef780 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe029784b80 10 client.0 _ll_register_callbacks cb 0x560bbdde6470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34316/0 (socket says 172.21.15.90:34316) 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 -- 172.21.15.90:0/3702451555 learned_addr learned my addr 172.21.15.90:0/3702451555 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdefdba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefaf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 -- 172.21.15.90:0/3702451555 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 msgr2=0x560bbdefdba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdefdba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 -- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 msgr2=0x560bbdefaf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefaf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.576+0000 7fe0177fe700 1 -- 172.21.15.90:0/3702451555 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe00c010c50 con 0x560bbdefe1f0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 secure :-1 s=READY pgs=889 cs=0 l=1 rev1=1 crypto rx=0x7fe010009850 tx=0x7fe010097b00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=426c62eaa9f3e204 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe01009f320 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe01009be40 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560bbde01380 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe01009b0a0 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 10 client.18018 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560bbdc6eb60 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560bbde01380 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe0100b6080 con 0x560bbdefe1f0 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 10 client.18018 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 10 client.18018 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 20 client.18018 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf0dbf0 0x560bbdf10010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 4 client.18018 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:21:03.584 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.577+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560bbde39700 con 0x560bbdf0dbf0 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.578+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf0dbf0 0x560bbdf10010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.578+0000 7fdffaff5700 10 client.18018 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.578+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf0dbf0 0x560bbdf10010 crc :-1 s=READY pgs=258 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1ae8d6676d054dac server_cookie=69249e6421a4a900 in_seq=0 out_seq=0 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fdffaff5700 1 -- 172.21.15.90:0/3702451555 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x560bbde39700 con 0x560bbdf0dbf0 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fdffaff5700 10 client.18018 handle_command_reply: tid=0 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 client.18018 shutdown 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 msgr2=0x560bbdefb530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 secure :-1 s=READY pgs=889 cs=0 l=1 rev1=1 crypto rx=0x7fe010009850 tx=0x7fe010097b00 comp rx=0 tx=0).stop 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 shutdown_connections 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdefb530 unknown :-1 s=CLOSED pgs=889 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf0dbf0 0x560bbdf10010 crc :-1 s=READY pgs=258 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdefdba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 --2- 172.21.15.90:0/3702451555 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdefaf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fdffaff5700 0 client.18018 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:03.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 >> 172.21.15.90:0/3702451555 conn(0x560bbde4e6a0 msgr2=0x560bbdee9c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 shutdown_connections 2024-04-06T14:21:03.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 1 -- 172.21.15.90:0/3702451555 wait complete. 2024-04-06T14:21:03.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.579+0000 7fe029784b80 20 client.18018 trim_cache size 0 max 16384 2024-04-06T14:21:03.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 Processor -- start 2024-04-06T14:21:03.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 -- start start 2024-04-06T14:21:03.602 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdf04e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdf09cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf0d9d0 con 0x560bbdefe1f0 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf053b0 con 0x560bbdef00a0 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe029784b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf0a210 con 0x560bbdeef780 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34320/0 (socket says 172.21.15.90:34320) 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 -- 172.21.15.90:0/3899010489 learned_addr learned my addr 172.21.15.90:0/3899010489 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdf04e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdf09cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 -- 172.21.15.90:0/3899010489 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 msgr2=0x560bbdf04e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdf04e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 -- 172.21.15.90:0/3899010489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 msgr2=0x560bbdf09cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.603 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdf09cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 -- 172.21.15.90:0/3899010489 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560bbde06ea0 con 0x560bbdefe1f0 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdf09cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.596+0000 7fe0177fe700 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fe0100981b0 tx=0x7fe01001dc70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af00f1ff921e730e server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fdffdffb700 1 -- 172.21.15.90:0/3899010489 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe01009ae30 con 0x560bbdefe1f0 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fdffdffb700 1 -- 172.21.15.90:0/3899010489 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe01009ba40 con 0x560bbdefe1f0 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fdffdffb700 1 -- 172.21.15.90:0/3899010489 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe01009f380 con 0x560bbdefe1f0 2024-04-06T14:21:03.604 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- 172.21.15.90:0/3899010489 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 msgr2=0x560bbdeea3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 secure :-1 s=READY pgs=890 cs=0 l=1 rev1=1 crypto rx=0x7fe0100981b0 tx=0x7fe01001dc70 comp rx=0 tx=0).stop 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- 172.21.15.90:0/3899010489 shutdown_connections 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdefe1f0 0x560bbdeea3a0 unknown :-1 s=CLOSED pgs=890 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdeef780 0x560bbdf04e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 --2- 172.21.15.90:0/3899010489 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdef00a0 0x560bbdf09cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- 172.21.15.90:0/3899010489 >> 172.21.15.90:0/3899010489 conn(0x560bbde4e6a0 msgr2=0x560bbdf095c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- 172.21.15.90:0/3899010489 shutdown_connections 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- 172.21.15.90:0/3899010489 wait complete. 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.597+0000 7fe029784b80 1 -- start start 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 Processor -- start 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdef00a0 0x560bbdf04d50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.605 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdefe1f0 0x560bbdf02d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf03450 con 0x560bbdeef780 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf035c0 con 0x560bbdefe1f0 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x560bbdf03780 con 0x560bbdef00a0 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 10 client.0 _ll_register_callbacks cb 0x560bbdde6470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34326/0 (socket says 172.21.15.90:34326) 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 -- 172.21.15.90:0/3207098903 learned_addr learned my addr 172.21.15.90:0/3207098903 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdef00a0 0x560bbdf04d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 -- 172.21.15.90:0/3207098903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdef00a0 msgr2=0x560bbdf04d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdef00a0 0x560bbdf04d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 -- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdefe1f0 msgr2=0x560bbdf02d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:03.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdefe1f0 0x560bbdf02d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 -- 172.21.15.90:0/3207098903 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560bbdc6eb60 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe016ffd700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 secure :-1 s=READY pgs=891 cs=0 l=1 rev1=1 crypto rx=0x7fe00c010f40 tx=0x7fe00c005750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=78d0710c161b95a3 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fdffaff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe00c02c040 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fe00c017070 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.598+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fe00c015070 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe00c011e10 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 10 client.18024 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x560bbdcc6cd0 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe00c015070 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fe00c039020 con 0x560bbdeef780 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 10 client.18024 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 10 client.18024 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 20 client.18024 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf62a40 0x560bbdf64e60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 4 client.18024 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "scrub status"}] 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x560bbdc8fcf0 con 0x560bbdf62a40 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.599+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf62a40 0x560bbdf64e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fdffaff5700 10 client.18024 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe0167fc700 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf62a40 0x560bbdf64e60 crc :-1 s=READY pgs=259 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ff32704a140e6cf1 server_cookie=f52ef9fe69854cb3 in_seq=0 out_seq=0 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fdffaff5700 1 -- 172.21.15.90:0/3207098903 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x560bbdc8fcf0 con 0x560bbdf62a40 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fdffaff5700 10 client.18024 handle_command_reply: tid=0 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 client.18024 shutdown 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 msgr2=0x560bbdf04810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 secure :-1 s=READY pgs=891 cs=0 l=1 rev1=1 crypto rx=0x7fe00c010f40 tx=0x7fe00c005750 comp rx=0 tx=0).stop 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 shutdown_connections 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x560bbdefe1f0 0x560bbdf02d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x560bbdeef780 0x560bbdf04810 unknown :-1 s=CLOSED pgs=891 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x560bbdf62a40 0x560bbdf64e60 crc :-1 s=READY pgs=259 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 --2- 172.21.15.90:0/3207098903 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x560bbdef00a0 0x560bbdf04d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fdffaff5700 0 client.18024 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 >> 172.21.15.90:0/3207098903 conn(0x560bbde4e6a0 msgr2=0x560bbdeef000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 shutdown_connections 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 1 -- 172.21.15.90:0/3207098903 wait complete. 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe029784b80 20 client.18024 trim_cache size 0 max 16384 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stdout: "status": "no active scrubs running", 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stdout: "scrubs": {} 2024-04-06T14:21:03.609 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:03.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 msgr2=0x7fe0000a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 0x7fe0000a3750 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7fe00c005950 tx=0x7fe00c00b410 comp rx=0 tx=0).stop 2024-04-06T14:21:03.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 msgr2=0x7fe01819c070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.600+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 secure :-1 s=READY pgs=887 cs=0 l=1 rev1=1 crypto rx=0x7fe010006130 tx=0x7fe010009300 comp rx=0 tx=0).stop 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 shutdown_connections 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fe0000a1290 0x7fe0000a3750 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fe01819bbf0 0x7fe01819c070 unknown :-1 s=CLOSED pgs=887 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fe018100f90 0x7fe018197360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 --2- 172.21.15.90:0/3270247850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe0181060b0 0x7fe0181978a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 >> 172.21.15.90:0/3270247850 conn(0x7fe0180fc7f0 msgr2=0x7fe0180feae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 shutdown_connections 2024-04-06T14:21:03.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.601+0000 7fe01d5d0700 1 -- 172.21.15.90:0/3270247850 wait complete. 2024-04-06T14:21:03.612 INFO:tasks.cephfs.filesystem:all active scrubs completed 2024-04-06T14:21:03.613 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:21:03.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:03 smithi134 ceph-mon[47549]: pgmap v490: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 393 KiB/s rd, 19 MiB/s wr, 1.85k op/s 2024-04-06T14:21:03.737 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 -- 172.21.15.90:0/187115423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 msgr2=0x7fb988102230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/187115423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb988102230 secure :-1 s=READY pgs=892 cs=0 l=1 rev1=1 crypto rx=0x7fb978009990 tx=0x7fb978009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 -- 172.21.15.90:0/187115423 shutdown_connections 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/187115423 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 0x7fb98810d6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/187115423 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb988102230 unknown :-1 s=CLOSED pgs=892 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/187115423 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb98810ac50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 -- 172.21.15.90:0/187115423 >> 172.21.15.90:0/187115423 conn(0x7fb9880fb650 msgr2=0x7fb9880fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 -- 172.21.15.90:0/187115423 shutdown_connections 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.735+0000 7fb98ca0f700 1 -- 172.21.15.90:0/187115423 wait complete. 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.736+0000 7fb98ca0f700 1 Processor -- start 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.736+0000 7fb98ca0f700 1 -- start start 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.736+0000 7fb98ca0f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.738 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.736+0000 7fb98ca0f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb988197600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 0x7fb98819b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb98819bfe0 con 0x7fb988101e50 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb98819c150 con 0x7fb988102770 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb98819c310 con 0x7fb98810b220 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34350/0 (socket says 172.21.15.90:34350) 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 -- 172.21.15.90:0/3710786481 learned_addr learned my addr 172.21.15.90:0/3710786481 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb986d9d700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 0x7fb98819b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb985d9b700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb988197600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 -- 172.21.15.90:0/3710786481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 msgr2=0x7fb98819b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.739 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 0x7fb98819b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 -- 172.21.15.90:0/3710786481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 msgr2=0x7fb988197600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb988197600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 -- 172.21.15.90:0/3710786481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9780095f0 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb985d9b700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb988197600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98659c700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7fb978009fd0 tx=0x7fb97800f670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb97801d070 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb978005510 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb98819c590 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb978004b30 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.737+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9881a3e30 con 0x7fb988101e50 2024-04-06T14:21:03.740 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.738+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb9881063a0 con 0x7fb988101e50 2024-04-06T14:21:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.739+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb97800fb90 con 0x7fb988101e50 2024-04-06T14:21:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.740+0000 7fb9777fe700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 0x7fb9700a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.740+0000 7fb985d9b700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 0x7fb9700a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.740+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fb9780cdef0 con 0x7fb988101e50 2024-04-06T14:21:03.741 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.740+0000 7fb985d9b700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 0x7fb9700a37d0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fb97c005950 tx=0x7fb97c0058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:03.742 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.743+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb978097470 con 0x7fb988101e50 2024-04-06T14:21:03.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.990+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb988069880 con 0x7fb988101e50 2024-04-06T14:21:03.990 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.991+0000 7fb9777fe700 1 -- 172.21.15.90:0/3710786481 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7fb97809aa90 con 0x7fb988101e50 2024-04-06T14:21:03.990 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:03.991 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 msgr2=0x7fb9700a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 0x7fb9700a37d0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fb97c005950 tx=0x7fb97c0058e0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 msgr2=0x7fb9881970c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 secure :-1 s=READY pgs=893 cs=0 l=1 rev1=1 crypto rx=0x7fb978009fd0 tx=0x7fb97800f670 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 shutdown_connections 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb9700a1310 0x7fb9700a37d0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb98810b220 0x7fb98819b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb988101e50 0x7fb9881970c0 unknown :-1 s=CLOSED pgs=893 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 --2- 172.21.15.90:0/3710786481 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb988102770 0x7fb988197600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:03.993 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 >> 172.21.15.90:0/3710786481 conn(0x7fb9880fb650 msgr2=0x7fb9880fcfd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:03.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 shutdown_connections 2024-04-06T14:21:03.994 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:03.993+0000 7fb98ca0f700 1 -- 172.21.15.90:0/3710786481 wait complete. 2024-04-06T14:21:03.994 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:21:04.005 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:21:04.006 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.130+0000 7f2f7d27f700 1 -- 172.21.15.90:0/1865907071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f78100820 msgr2=0x7f2f78104860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.130+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/1865907071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f78100820 0x7f2f78104860 secure :-1 s=READY pgs=894 cs=0 l=1 rev1=1 crypto rx=0x7f2f6c009990 tx=0x7f2f6c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- 172.21.15.90:0/1865907071 shutdown_connections 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/1865907071 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 0x7f2f7810cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/1865907071 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 0x7f2f78100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/1865907071 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f78100820 0x7f2f78104860 unknown :-1 s=CLOSED pgs=894 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- 172.21.15.90:0/1865907071 >> 172.21.15.90:0/1865907071 conn(0x7f2f78076d50 msgr2=0x7f2f78077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- 172.21.15.90:0/1865907071 shutdown_connections 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- 172.21.15.90:0/1865907071 wait complete. 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 Processor -- start 2024-04-06T14:21:04.133 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- start start 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 0x7f2f78197260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 0x7f2f781977a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7819c5c0 con 0x7f2f7819bb40 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.131+0000 7f2f7d27f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7819c730 con 0x7f2f780ffe70 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f7d27f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7819c8f0 con 0x7f2f78104f90 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34372/0 (socket says 172.21.15.90:34372) 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 -- 172.21.15.90:0/4274504425 learned_addr learned my addr 172.21.15.90:0/4274504425 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 0x7f2f78197260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f767fc700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 0x7f2f781977a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.134 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 -- 172.21.15.90:0/4274504425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 msgr2=0x7f2f781977a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 0x7f2f781977a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 -- 172.21.15.90:0/4274504425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 msgr2=0x7f2f78197260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 0x7f2f78197260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 -- 172.21.15.90:0/4274504425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f6c0095f0 con 0x7f2f7819bb40 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f777fe700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7f2f70007ee0 tx=0x7f2f7000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f5ffff700 1 -- 172.21.15.90:0/4274504425 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f70017070 con 0x7f2f7819bb40 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f781a3e50 con 0x7f2f7819bb40 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f5ffff700 1 -- 172.21.15.90:0/4274504425 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f7000f040 con 0x7f2f7819bb40 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f5ffff700 1 -- 172.21.15.90:0/4274504425 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f70005550 con 0x7f2f7819bb40 2024-04-06T14:21:04.135 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.132+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f781a4380 con 0x7f2f7819bb40 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.134+0000 7f2f5ffff700 1 -- 172.21.15.90:0/4274504425 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f2f7000db20 con 0x7f2f7819bb40 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.135+0000 7f2f5ffff700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 0x7f2f600a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.135+0000 7f2f5ffff700 1 -- 172.21.15.90:0/4274504425 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2f700cc380 con 0x7f2f7819bb40 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.135+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 0x7f2f600a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.135+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 0x7f2f600a3750 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f2f68005950 tx=0x7f2f6800b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.142 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 Processor -- start 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 -- start start 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 0x55ada86f4b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fe8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada864c990 con 0x55ada86fede0 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada864a350 con 0x55ada86f4780 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f89433b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada8645c40 con 0x55ada86f50a0 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34380/0 (socket says 172.21.15.90:34380) 2024-04-06T14:21:04.143 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 -- 172.21.15.90:0/3662249787 learned_addr learned my addr 172.21.15.90:0/3662249787 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 0x55ada86f4b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 -- 172.21.15.90:0/3662249787 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 msgr2=0x55ada86fe8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fe8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 -- 172.21.15.90:0/3662249787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 msgr2=0x55ada86f4b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 0x55ada86f4b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.136+0000 7f2f777fe700 1 -- 172.21.15.90:0/3662249787 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ada84c73b0 con 0x55ada86fede0 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 0x55ada86f4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f777fe700 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 secure :-1 s=READY pgs=896 cs=0 l=1 rev1=1 crypto rx=0x7f2f70095970 tx=0x7f2f70004b50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=dd73190669170305 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f5d7fa700 1 -- 172.21.15.90:0/3662249787 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f7009d2a0 con 0x55ada86fede0 2024-04-06T14:21:04.144 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f5d7fa700 1 -- 172.21.15.90:0/3662249787 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f700a18f0 con 0x55ada86fede0 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f5d7fa700 1 -- 172.21.15.90:0/3662249787 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f70022250 con 0x55ada86fede0 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 -- 172.21.15.90:0/3662249787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 msgr2=0x55ada87011d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 secure :-1 s=READY pgs=896 cs=0 l=1 rev1=1 crypto rx=0x7f2f70095970 tx=0x7f2f70004b50 comp rx=0 tx=0).stop 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 -- 172.21.15.90:0/3662249787 shutdown_connections 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86fede0 0x55ada87011d0 unknown :-1 s=CLOSED pgs=896 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f4780 0x55ada86f4b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 --2- 172.21.15.90:0/3662249787 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fe8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 -- 172.21.15.90:0/3662249787 >> 172.21.15.90:0/3662249787 conn(0x55ada86536a0 msgr2=0x55ada86efa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.137+0000 7f2f89433b80 1 -- 172.21.15.90:0/3662249787 shutdown_connections 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 -- 172.21.15.90:0/3662249787 wait complete. 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 -- start start 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 Processor -- start 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.145 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86fc6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada85e1d80 con 0x55ada86f4780 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada85e1a20 con 0x55ada86fede0 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada84c96a0 con 0x55ada86f50a0 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f89433b80 10 client.0 _ll_register_callbacks cb 0x55ada85eb470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.138+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.146 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34394/0 (socket says 172.21.15.90:34394) 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 -- 172.21.15.90:0/2619896194 learned_addr learned my addr 172.21.15.90:0/2619896194 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f767fc700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f777fe700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86fc6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 -- 172.21.15.90:0/2619896194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 msgr2=0x55ada86fc180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 -- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 msgr2=0x55ada86fc6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86fc6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 -- 172.21.15.90:0/2619896194 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f781a4380 con 0x55ada86f4780 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7f2f68011770 tx=0x7f2f680047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2af0fad63a04b334 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f5aff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f68004a30 con 0x55ada86f4780 2024-04-06T14:21:04.147 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2f68017070 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ada8606380 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2f68010ea0 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f89433b80 10 client.18042 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ada8473b60 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.139+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ada8606380 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2f6801f640 con 0x55ada86f4780 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 10 client.18042 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 10 client.18042 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 20 client.18042 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada870ebb0 0x55ada8710fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.148 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 4 client.18042 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ada863e700 con 0x55ada870ebb0 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f767fc700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada870ebb0 0x55ada8710fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f767fc700 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada870ebb0 0x55ada8710fd0 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f717f2cb2e2e20eb server_cookie=ec0c94b5abbfa13a in_seq=0 out_seq=0 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.140+0000 7f2f5aff5700 10 client.18042 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f5aff5700 1 -- 172.21.15.90:0/2619896194 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55ada863e700 con 0x55ada870ebb0 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f5aff5700 10 client.18042 handle_command_reply: tid=0 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 client.18042 shutdown 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 msgr2=0x55ada86fbc40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 secure :-1 s=READY pgs=897 cs=0 l=1 rev1=1 crypto rx=0x7f2f68011770 tx=0x7f2f680047c0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.149 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 shutdown_connections 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada870ebb0 0x55ada8710fd0 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86fc6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f5aff5700 0 client.18042 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fbc40 unknown :-1 s=CLOSED pgs=897 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 --2- 172.21.15.90:0/2619896194 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 >> 172.21.15.90:0/2619896194 conn(0x55ada86536a0 msgr2=0x55ada86eec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 shutdown_connections 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 1 -- 172.21.15.90:0/2619896194 wait complete. 2024-04-06T14:21:04.150 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.141+0000 7f2f89433b80 20 client.18042 trim_cache size 0 max 16384 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 Processor -- start 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 -- start start 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86f8ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada86f1220 con 0x55ada86f4780 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada870b500 con 0x55ada86fede0 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f89433b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada86fa100 con 0x55ada86f50a0 2024-04-06T14:21:04.159 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.154+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34406/0 (socket says 172.21.15.90:34406) 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 -- 172.21.15.90:0/1589285441 learned_addr learned my addr 172.21.15.90:0/1589285441 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 -- 172.21.15.90:0/1589285441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 msgr2=0x55ada86fc4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f777fe700 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86f8ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 -- 172.21.15.90:0/1589285441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 msgr2=0x55ada86f8ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86f8ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 -- 172.21.15.90:0/1589285441 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ada860bea0 con 0x55ada86f4780 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7f2f68000c00 tx=0x7f2f6800b500 comp rx=0 tx=0).ready entity=mon.0 client_cookie=30501186f01ddae1 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f5dffb700 1 -- 172.21.15.90:0/1589285441 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f6801fbe0 con 0x55ada86f4780 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f5dffb700 1 -- 172.21.15.90:0/1589285441 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f68015070 con 0x55ada86f4780 2024-04-06T14:21:04.160 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.155+0000 7f2f5dffb700 1 -- 172.21.15.90:0/1589285441 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f680053a0 con 0x55ada86f4780 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- 172.21.15.90:0/1589285441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 msgr2=0x55ada86fac20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 secure :-1 s=READY pgs=898 cs=0 l=1 rev1=1 crypto rx=0x7f2f68000c00 tx=0x7f2f6800b500 comp rx=0 tx=0).stop 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- 172.21.15.90:0/1589285441 shutdown_connections 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86fede0 0x55ada86f8ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada86fac20 unknown :-1 s=CLOSED pgs=898 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- 172.21.15.90:0/1589285441 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86f50a0 0x55ada86fc4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- 172.21.15.90:0/1589285441 >> 172.21.15.90:0/1589285441 conn(0x55ada86536a0 msgr2=0x55ada86fa510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- 172.21.15.90:0/1589285441 shutdown_connections 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- 172.21.15.90:0/1589285441 wait complete. 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- start start 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 Processor -- start 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.161 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f50a0 0x55ada8703bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86fede0 0x55ada8704130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada8706030 con 0x55ada86f4780 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada8704670 con 0x55ada86f50a0 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ada8704830 con 0x55ada86fede0 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f89433b80 10 client.0 _ll_register_callbacks cb 0x55ada85eb470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f76ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34412/0 (socket says 172.21.15.90:34412) 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.156+0000 7f2f76ffd700 1 -- 172.21.15.90:0/323469560 learned_addr learned my addr 172.21.15.90:0/323469560 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f767fc700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f50a0 0x55ada8703bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 -- 172.21.15.90:0/323469560 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86fede0 msgr2=0x55ada8704130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86fede0 0x55ada8704130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 -- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f50a0 msgr2=0x55ada8703bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.162 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f50a0 0x55ada8703bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 -- 172.21.15.90:0/323469560 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ada8473b60 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f76ffd700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 secure :-1 s=READY pgs=899 cs=0 l=1 rev1=1 crypto rx=0x7f2f68022040 tx=0x7f2f68005a00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=813f9c66696db79a server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f5aff5700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f68004d00 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2f68017070 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2f700a18f0 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2f6801f640 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f89433b80 10 client.18048 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55ada84cbcd0 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.157+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f6804c020 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f2f6803e070 con 0x55ada86f4780 2024-04-06T14:21:04.163 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 10 client.18048 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 10 client.18048 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.h 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 20 client.18048 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada8763a10 0x55ada8765e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 4 client.18048 mds_command: new command op to 24485 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 --> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ada8494cf0 con 0x55ada8763a10 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f767fc700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada8763a10 0x55ada8765e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f5aff5700 10 client.18048 ms_handle_connect on v2:172.21.15.105:6836/2851238848 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f767fc700 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada8763a10 0x55ada8765e30 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=9a6267877b6ead92 server_cookie=3e429233100f608d in_seq=0 out_seq=0 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f5aff5700 1 -- 172.21.15.90:0/323469560 <== mds.2 v2:172.21.15.105:6836/2851238848 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ada8494cf0 con 0x55ada8763a10 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f5aff5700 10 client.18048 handle_command_reply: tid=0 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 client.18048 shutdown 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 msgr2=0x55ada8705750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 secure :-1 s=READY pgs=899 cs=0 l=1 rev1=1 crypto rx=0x7f2f68022040 tx=0x7f2f68005a00 comp rx=0 tx=0).stop 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 shutdown_connections 2024-04-06T14:21:04.164 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:6836/2851238848,v1:172.21.15.105:6837/2851238848] conn(0x55ada8763a10 0x55ada8765e30 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.158+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55ada86fede0 0x55ada8704130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f5aff5700 0 client.18048 ms_handle_reset on v2:172.21.15.105:6836/2851238848 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55ada86f4780 0x55ada8705750 unknown :-1 s=CLOSED pgs=899 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 1 --2- 172.21.15.90:0/323469560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55ada86f50a0 0x55ada8703bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 >> 172.21.15.90:0/323469560 conn(0x55ada86536a0 msgr2=0x55ada86f4000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 shutdown_connections 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 1 -- 172.21.15.90:0/323469560 wait complete. 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f89433b80 20 client.18048 trim_cache size 0 max 16384 2024-04-06T14:21:04.165 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:21:04.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 msgr2=0x7f2f600a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 0x7f2f600a3750 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f2f68005950 tx=0x7f2f6800b410 comp rx=0 tx=0).stop 2024-04-06T14:21:04.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 msgr2=0x7f2f7819bfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.166 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 secure :-1 s=READY pgs=895 cs=0 l=1 rev1=1 crypto rx=0x7f2f70007ee0 tx=0x7f2f7000d3b0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 shutdown_connections 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2f600a1290 0x7f2f600a3750 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2f7819bb40 0x7f2f7819bfc0 unknown :-1 s=CLOSED pgs=895 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f780ffe70 0x7f2f78197260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 --2- 172.21.15.90:0/4274504425 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2f78104f90 0x7f2f781977a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 >> 172.21.15.90:0/4274504425 conn(0x7f2f78076d50 msgr2=0x7f2f78069700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 shutdown_connections 2024-04-06T14:21:04.167 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.159+0000 7f2f7d27f700 1 -- 172.21.15.90:0/4274504425 wait complete. 2024-04-06T14:21:04.171 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:21:04.171 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2024-04-06T14:21:04.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:04 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3710786481' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:04.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 -- 172.21.15.90:0/3384471209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c100820 msgr2=0x7f384c104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 --2- 172.21.15.90:0/3384471209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c100820 0x7f384c104840 secure :-1 s=READY pgs=900 cs=0 l=1 rev1=1 crypto rx=0x7f383c009990 tx=0x7f383c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 -- 172.21.15.90:0/3384471209 shutdown_connections 2024-04-06T14:21:04.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 --2- 172.21.15.90:0/3384471209 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 0x7f384c10caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.299 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 --2- 172.21.15.90:0/3384471209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c0ffe70 0x7f384c100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 --2- 172.21.15.90:0/3384471209 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c100820 0x7f384c104840 unknown :-1 s=CLOSED pgs=900 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 -- 172.21.15.90:0/3384471209 >> 172.21.15.90:0/3384471209 conn(0x7f384c076d50 msgr2=0x7f384c077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.297+0000 7f3853a59700 1 -- 172.21.15.90:0/3384471209 shutdown_connections 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 -- 172.21.15.90:0/3384471209 wait complete. 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 Processor -- start 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 -- start start 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 0x7f384c101e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c102360 0x7f384c1a5360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f384c1a59a0 con 0x7f384c0ffe70 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f384c1a5b10 con 0x7f384c102360 2024-04-06T14:21:04.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3853a59700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f384c1a5cd0 con 0x7f384c104f70 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f38517f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f38517f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34430/0 (socket says 172.21.15.90:34430) 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f38517f5700 1 -- 172.21.15.90:0/2773294883 learned_addr learned my addr 172.21.15.90:0/2773294883 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.298+0000 7f3851ff6700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c102360 0x7f384c1a5360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 -- 172.21.15.90:0/2773294883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 msgr2=0x7f384c101e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 0x7f384c101e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 0x7f384c101e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 -- 172.21.15.90:0/2773294883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c102360 msgr2=0x7f384c1a5360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c102360 0x7f384c1a5360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 -- 172.21.15.90:0/2773294883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f383c0095f0 con 0x7f384c0ffe70 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38517f5700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 secure :-1 s=READY pgs=901 cs=0 l=1 rev1=1 crypto rx=0x7f384800e9a0 tx=0x7f384800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38427fc700 1 -- 172.21.15.90:0/2773294883 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3848009790 con 0x7f384c0ffe70 2024-04-06T14:21:04.301 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38427fc700 1 -- 172.21.15.90:0/2773294883 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f384800cbd0 con 0x7f384c0ffe70 2024-04-06T14:21:04.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f384c1a5fb0 con 0x7f384c0ffe70 2024-04-06T14:21:04.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f38427fc700 1 -- 172.21.15.90:0/2773294883 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3848018680 con 0x7f384c0ffe70 2024-04-06T14:21:04.302 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.299+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f384c1a6430 con 0x7f384c0ffe70 2024-04-06T14:21:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.301+0000 7f38427fc700 1 -- 172.21.15.90:0/2773294883 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f384800cd40 con 0x7f384c0ffe70 2024-04-06T14:21:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.302+0000 7f38427fc700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 0x7f38380a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.302+0000 7f38427fc700 1 -- 172.21.15.90:0/2773294883 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f3848014070 con 0x7f384c0ffe70 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.302+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 0x7f38380a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.302+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 0x7f38380a3750 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f383c000c00 tx=0x7f383c005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 Processor -- start 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 -- start start 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c7b8a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c7e1d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786bc9990 con 0x55d786c71780 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786bc7350 con 0x55d786c7bde0 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f385fc0db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786bc2c40 con 0x55d786c720a0 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34438/0 (socket says 172.21.15.90:34438) 2024-04-06T14:21:04.310 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 -- 172.21.15.90:0/1820115768 learned_addr learned my addr 172.21.15.90:0/1820115768 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f3851ff6700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c7e1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f3850ff4700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c7b8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 -- 172.21.15.90:0/1820115768 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 msgr2=0x55d786c7b8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c7b8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 -- 172.21.15.90:0/1820115768 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 msgr2=0x55d786c7e1d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c7e1d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 -- 172.21.15.90:0/1820115768 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d786a443b0 con 0x55d786c71780 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f3851ff6700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c7e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f3850ff4700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c7b8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.303+0000 7f38517f5700 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f38480a1fb0 tx=0x7f3848099a40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b08b61295b0514c7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f38377fe700 1 -- 172.21.15.90:0/1820115768 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f384809de90 con 0x55d786c71780 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f38377fe700 1 -- 172.21.15.90:0/1820115768 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f38480a24e0 con 0x55d786c71780 2024-04-06T14:21:04.311 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f38377fe700 1 -- 172.21.15.90:0/1820115768 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3848099750 con 0x55d786c71780 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 -- 172.21.15.90:0/1820115768 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 msgr2=0x55d786c71b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 secure :-1 s=READY pgs=902 cs=0 l=1 rev1=1 crypto rx=0x7f38480a1fb0 tx=0x7f3848099a40 comp rx=0 tx=0).stop 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 -- 172.21.15.90:0/1820115768 shutdown_connections 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c7e1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c71780 0x55d786c71b60 unknown :-1 s=CLOSED pgs=902 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1820115768 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c7b8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 -- 172.21.15.90:0/1820115768 >> 172.21.15.90:0/1820115768 conn(0x55d786bd06a0 msgr2=0x55d786c6ca80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 -- 172.21.15.90:0/1820115768 shutdown_connections 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.304+0000 7f385fc0db80 1 -- 172.21.15.90:0/1820115768 wait complete. 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 -- start start 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 Processor -- start 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c7b840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.312 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c7bde0 0x55d786c79200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786b5ed80 con 0x55d786c720a0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786b5ea20 con 0x55d786c71780 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786a466a0 con 0x55d786c7bde0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f385fc0db80 10 client.0 _ll_register_callbacks cb 0x55d786b68470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34450/0 (socket says 172.21.15.90:34450) 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 -- 172.21.15.90:0/1671318697 learned_addr learned my addr 172.21.15.90:0/1671318697 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f38517f5700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c7b840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3851ff6700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c7bde0 0x55d786c79200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 -- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c7bde0 msgr2=0x55d786c79200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c7bde0 0x55d786c79200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 -- 172.21.15.90:0/1671318697 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 msgr2=0x55d786c7b840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.313 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c7b840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.305+0000 7f3850ff4700 1 -- 172.21.15.90:0/1671318697 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f384c1a6430 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3850ff4700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f384c06a4e0 tx=0x7f383c009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f3295d33f4aa5897 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f383c02b070 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f383c0048e0 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d786b83380 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f383c01e070 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 10 client.18060 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d7869f0b60 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f383c03c080 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f383c00fc80 con 0x55d786c720a0 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 10 client.18060 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 10 client.18060 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:21:04.314 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 20 client.18060 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786c8b890 0x55d786c8dcb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 4 client.18060 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.306+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d786bbb700 con 0x55d786c8b890 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.307+0000 7f38517f5700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786c8b890 0x55d786c8dcb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.307+0000 7f3834ff9700 10 client.18060 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.307+0000 7f38517f5700 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786c8b890 0x55d786c8dcb0 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=455d9c1573ef6c77 server_cookie=1c16c8cf475898f2 in_seq=0 out_seq=0 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f3834ff9700 1 -- 172.21.15.90:0/1671318697 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x55d786bbb700 con 0x55d786c8b890 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f3834ff9700 10 client.18060 handle_command_reply: tid=0 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 client.18060 shutdown 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 msgr2=0x55d786c78ac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 secure :-1 s=READY pgs=903 cs=0 l=1 rev1=1 crypto rx=0x7f384c06a4e0 tx=0x7f383c009310 comp rx=0 tx=0).stop 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 shutdown_connections 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786c8b890 0x55d786c8dcb0 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c7bde0 0x55d786c79200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.315 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c7b840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 --2- 172.21.15.90:0/1671318697 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c78ac0 unknown :-1 s=CLOSED pgs=903 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 >> 172.21.15.90:0/1671318697 conn(0x55d786bd06a0 msgr2=0x55d786c6bc20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f3834ff9700 0 client.18060 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 shutdown_connections 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 1 -- 172.21.15.90:0/1671318697 wait complete. 2024-04-06T14:21:04.316 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.308+0000 7f385fc0db80 20 client.18060 trim_cache size 0 max 16384 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 Processor -- start 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 -- start start 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c791b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c829a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c79f50 con 0x55d786c7bde0 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c797e0 con 0x55d786c71780 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f385fc0db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c6ddc0 con 0x55d786c720a0 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34458/0 (socket says 172.21.15.90:34458) 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 -- 172.21.15.90:0/580791571 learned_addr learned my addr 172.21.15.90:0/580791571 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.327 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f38517f5700 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c791b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 -- 172.21.15.90:0/580791571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 msgr2=0x55d786c829a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c829a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 -- 172.21.15.90:0/580791571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 msgr2=0x55d786c791b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c791b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f3851ff6700 1 -- 172.21.15.90:0/580791571 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d786b88ea0 con 0x55d786c7bde0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.321+0000 7f38517f5700 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c791b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f3851ff6700 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 secure :-1 s=READY pgs=904 cs=0 l=1 rev1=1 crypto rx=0x7f38440134c0 tx=0x7f3844007950 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52032622d16e49d7 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f3837fff700 1 -- 172.21.15.90:0/580791571 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3844013e90 con 0x55d786c7bde0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f3837fff700 1 -- 172.21.15.90:0/580791571 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3844004bb0 con 0x55d786c7bde0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f3837fff700 1 -- 172.21.15.90:0/580791571 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f38440051c0 con 0x55d786c7bde0 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 -- 172.21.15.90:0/580791571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 msgr2=0x55d786c6c520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 secure :-1 s=READY pgs=904 cs=0 l=1 rev1=1 crypto rx=0x7f38440134c0 tx=0x7f3844007950 comp rx=0 tx=0).stop 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 -- 172.21.15.90:0/580791571 shutdown_connections 2024-04-06T14:21:04.328 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c7bde0 0x55d786c6c520 unknown :-1 s=CLOSED pgs=904 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c71780 0x55d786c791b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 --2- 172.21.15.90:0/580791571 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c720a0 0x55d786c829a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 -- 172.21.15.90:0/580791571 >> 172.21.15.90:0/580791571 conn(0x55d786bd06a0 msgr2=0x55d786c73e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 -- 172.21.15.90:0/580791571 shutdown_connections 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.322+0000 7f385fc0db80 1 -- 172.21.15.90:0/580791571 wait complete. 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 -- start start 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 Processor -- start 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c71780 0x55d786c82100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c80980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c81020 con 0x55d786c720a0 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c81190 con 0x55d786c7bde0 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d786c81350 con 0x55d786c71780 2024-04-06T14:21:04.329 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f385fc0db80 10 client.0 _ll_register_callbacks cb 0x55d786b68470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34464/0 (socket says 172.21.15.90:34464) 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 -- 172.21.15.90:0/2862953518 learned_addr learned my addr 172.21.15.90:0/2862953518 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3851ff6700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c80980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f38517f5700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c71780 0x55d786c82100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 -- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c71780 msgr2=0x55d786c82100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c71780 0x55d786c82100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 -- 172.21.15.90:0/2862953518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 msgr2=0x55d786c80980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c80980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.323+0000 7f3850ff4700 1 -- 172.21.15.90:0/2862953518 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3844007e70 con 0x55d786c720a0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3850ff4700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7f383c009990 tx=0x7f383c039c30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e04addd8a52da249 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.330 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f383c02b040 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f383c004ab0 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f38480a24e0 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7f383c004c70 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 10 client.18066 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x55d786a48cd0 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f383c047020 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7f383c030420 con 0x55d786c720a0 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 10 client.18066 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 10 client.18066 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.i 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 20 client.18066 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786ce0510 0x55d786ce2930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 4 client.18066 mds_command: new command op to 24490 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.324+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 --> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d786a11cf0 con 0x55d786ce0510 2024-04-06T14:21:04.331 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f38517f5700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786ce0510 0x55d786ce2930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f3834ff9700 10 client.18066 ms_handle_connect on v2:172.21.15.134:6838/1913582469 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f38517f5700 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786ce0510 0x55d786ce2930 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=527a3691e141d767 server_cookie=4be5ad49f81249d7 in_seq=0 out_seq=0 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f3834ff9700 1 -- 172.21.15.90:0/2862953518 <== mds.1 v2:172.21.15.134:6838/1913582469 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d786a11cf0 con 0x55d786ce0510 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f3834ff9700 10 client.18066 handle_command_reply: tid=0 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 client.18066 shutdown 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 msgr2=0x55d786c82640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 secure :-1 s=READY pgs=905 cs=0 l=1 rev1=1 crypto rx=0x7f383c009990 tx=0x7f383c039c30 comp rx=0 tx=0).stop 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 shutdown_connections 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:6838/1913582469,v1:172.21.15.134:6839/1913582469] conn(0x55d786ce0510 0x55d786ce2930 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x55d786c7bde0 0x55d786c80980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f3834ff9700 0 client.18066 ms_handle_reset on v2:172.21.15.134:6838/1913582469 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x55d786c71780 0x55d786c82100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 --2- 172.21.15.90:0/2862953518 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x55d786c720a0 0x55d786c82640 unknown :-1 s=CLOSED pgs=905 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 >> 172.21.15.90:0/2862953518 conn(0x55d786bd06a0 msgr2=0x55d786c6c410 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.332 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 shutdown_connections 2024-04-06T14:21:04.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 1 -- 172.21.15.90:0/2862953518 wait complete. 2024-04-06T14:21:04.333 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.325+0000 7f385fc0db80 20 client.18066 trim_cache size 0 max 16384 2024-04-06T14:21:04.333 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 msgr2=0x7f38380a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 0x7f38380a3750 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f383c000c00 tx=0x7f383c005fb0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 msgr2=0x7f384c1018e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 secure :-1 s=READY pgs=901 cs=0 l=1 rev1=1 crypto rx=0x7f384800e9a0 tx=0x7f384800ecb0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 shutdown_connections 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f38380a1290 0x7f38380a3750 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f384c102360 0x7f384c1a5360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f384c0ffe70 0x7f384c1018e0 unknown :-1 s=CLOSED pgs=901 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.334 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 --2- 172.21.15.90:0/2773294883 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f384c104f70 0x7f384c101e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 >> 172.21.15.90:0/2773294883 conn(0x7f384c076d50 msgr2=0x7f384c068180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 shutdown_connections 2024-04-06T14:21:04.335 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.326+0000 7f3853a59700 1 -- 172.21.15.90:0/2773294883 wait complete. 2024-04-06T14:21:04.338 INFO:teuthology.orchestra.run:Running command with timeout 120 2024-04-06T14:21:04.338 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 -- 172.21.15.90:0/4205936100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 msgr2=0x7fa9a4100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 --2- 172.21.15.90:0/4205936100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a4100250 secure :-1 s=READY pgs=906 cs=0 l=1 rev1=1 crypto rx=0x7fa994009990 tx=0x7fa994009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 -- 172.21.15.90:0/4205936100 shutdown_connections 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 --2- 172.21.15.90:0/4205936100 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 0x7fa9a410caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 --2- 172.21.15.90:0/4205936100 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a4100250 unknown :-1 s=CLOSED pgs=906 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 --2- 172.21.15.90:0/4205936100 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4100820 0x7fa9a4104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.467 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 -- 172.21.15.90:0/4205936100 >> 172.21.15.90:0/4205936100 conn(0x7fa9a4076d50 msgr2=0x7fa9a4077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 -- 172.21.15.90:0/4205936100 shutdown_connections 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.465+0000 7fa9aa667700 1 -- 172.21.15.90:0/4205936100 wait complete. 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 Processor -- start 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 -- start start 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 0x7fa9a4198630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.468 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 0x7fa9a41a3170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a41a3810 con 0x7fa9a40ffe70 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a41a3980 con 0x7fa9a4198b70 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9aa667700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a41a3b40 con 0x7fa9a4104f70 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34490/0 (socket says 172.21.15.90:34490) 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 -- 172.21.15.90:0/2146930548 learned_addr learned my addr 172.21.15.90:0/2146930548 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.469 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 0x7fa9a41a3170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 0x7fa9a4198630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 -- 172.21.15.90:0/2146930548 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 msgr2=0x7fa9a4198630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 0x7fa9a4198630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 -- 172.21.15.90:0/2146930548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 msgr2=0x7fa9a41a3170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 0x7fa9a41a3170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.470 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 -- 172.21.15.90:0/2146930548 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9940095f0 con 0x7fa9a40ffe70 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 0x7fa9a41a3170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.466+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7fa994005910 tx=0x7fa994009570 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.467+0000 7fa9a17fa700 1 -- 172.21.15.90:0/2146930548 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa99401d070 con 0x7fa9a40ffe70 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.467+0000 7fa9a17fa700 1 -- 172.21.15.90:0/2146930548 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa994022470 con 0x7fa9a40ffe70 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.467+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9a41a3dc0 con 0x7fa9a40ffe70 2024-04-06T14:21:04.471 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.467+0000 7fa9a17fa700 1 -- 172.21.15.90:0/2146930548 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa994017650 con 0x7fa9a40ffe70 2024-04-06T14:21:04.472 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.467+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9a41a41e0 con 0x7fa9a40ffe70 2024-04-06T14:21:04.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.469+0000 7fa9a17fa700 1 -- 172.21.15.90:0/2146930548 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fa9940225e0 con 0x7fa9a40ffe70 2024-04-06T14:21:04.485 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.470+0000 7fa9a17fa700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 0x7fa98c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.470+0000 7fa9a17fa700 1 -- 172.21.15.90:0/2146930548 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa9940ce0f0 con 0x7fa9a40ffe70 2024-04-06T14:21:04.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.470+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 0x7fa98c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.470+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 0x7fa98c0a37d0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fa9a40685f0 tx=0x7fa998009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 Processor -- start 2024-04-06T14:21:04.486 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 -- start start 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee162cb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee1655e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee0ac990 con 0x563eee154780 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee0aa350 con 0x563eee1550a0 2024-04-06T14:21:04.487 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee0a5c40 con 0x563eee1631f0 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34496/0 (socket says 172.21.15.90:34496) 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 -- 172.21.15.90:0/3327180539 learned_addr learned my addr 172.21.15.90:0/3327180539 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee162cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee1655e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.488 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 -- 172.21.15.90:0/3327180539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 msgr2=0x563eee1655e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee1655e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 -- 172.21.15.90:0/3327180539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 msgr2=0x563eee162cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee162cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 -- 172.21.15.90:0/3327180539 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563eedf273b0 con 0x563eee154780 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee162cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:04.489 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 secure :-1 s=READY pgs=908 cs=0 l=1 rev1=1 crypto rx=0x7fa994097770 tx=0x7fa994009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=66d73abadd461b24 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.471+0000 7fa98a7fc700 1 -- 172.21.15.90:0/3327180539 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa99409b270 con 0x563eee154780 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa98a7fc700 1 -- 172.21.15.90:0/3327180539 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa99409f0a0 con 0x563eee154780 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa98a7fc700 1 -- 172.21.15.90:0/3327180539 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa99409b760 con 0x563eee154780 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3327180539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 msgr2=0x563eee154b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 secure :-1 s=READY pgs=908 cs=0 l=1 rev1=1 crypto rx=0x7fa994097770 tx=0x7fa994009310 comp rx=0 tx=0).stop 2024-04-06T14:21:04.490 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3327180539 shutdown_connections 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee1655e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee154b60 unknown :-1 s=CLOSED pgs=908 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3327180539 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee162cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3327180539 >> 172.21.15.90:0/3327180539 conn(0x563eee0b36a0 msgr2=0x563eee14fa80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3327180539 shutdown_connections 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.472+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3327180539 wait complete. 2024-04-06T14:21:04.491 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 -- start start 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 Processor -- start 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee154780 0x563eee162b50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1631f0 0x563eee160420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee041d80 con 0x563eee1550a0 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee041a20 con 0x563eee1631f0 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eedf296a0 con 0x563eee154780 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9b681bb80 10 client.0 _ll_register_callbacks cb 0x563eee04b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.492 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34508/0 (socket says 172.21.15.90:34508) 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 -- 172.21.15.90:0/2324552625 learned_addr learned my addr 172.21.15.90:0/2324552625 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee154780 0x563eee162b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1631f0 0x563eee160420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 -- 172.21.15.90:0/2324552625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee154780 msgr2=0x563eee162b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee154780 0x563eee162b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 -- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1631f0 msgr2=0x563eee160420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.493 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1631f0 0x563eee160420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 -- 172.21.15.90:0/2324552625 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9a41a41e0 con 0x563eee1550a0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 secure :-1 s=READY pgs=909 cs=0 l=1 rev1=1 crypto rx=0x7fa998012850 tx=0x7fa9980047c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d0b47516ac17c292 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.473+0000 7fa9837fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa99801f780 con 0x563eee1550a0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa998018070 con 0x563eee1550a0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563eee066380 con 0x563eee1550a0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa99801f8e0 con 0x563eee1550a0 2024-04-06T14:21:04.494 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 10 client.18078 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563eeded3b60 con 0x563eee1550a0 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa998035080 con 0x563eee1550a0 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa998029a30 con 0x563eee1550a0 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 10 client.18078 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 10 client.18078 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:21:04.495 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 20 client.18078 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee172bd0 0x563eee174ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 4 client.18078 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "get_command_descriptions"}] 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563eee09e700 con 0x563eee172bd0 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.474+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee172bd0 0x563eee174ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.475+0000 7fa9837fe700 10 client.18078 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.475+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee172bd0 0x563eee174ff0 crc :-1 s=READY pgs=260 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=297e8d12c0c5ecb5 server_cookie=56ebad25b9bbdebe in_seq=0 out_seq=0 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.475+0000 7fa9837fe700 1 -- 172.21.15.90:0/2324552625 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18209 (crc 0 0 0) 0x563eee09e700 con 0x563eee172bd0 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.475+0000 7fa9837fe700 10 client.18078 handle_command_reply: tid=0 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 client.18078 shutdown 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 msgr2=0x563eee15fe40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 secure :-1 s=READY pgs=909 cs=0 l=1 rev1=1 crypto rx=0x7fa998012850 tx=0x7fa9980047c0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.496 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 shutdown_connections 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1631f0 0x563eee160420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee172bd0 0x563eee174ff0 crc :-1 s=READY pgs=260 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee154780 0x563eee162b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/2324552625 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee15fe40 unknown :-1 s=CLOSED pgs=909 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9837fe700 0 client.18078 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 >> 172.21.15.90:0/2324552625 conn(0x563eee0b36a0 msgr2=0x563eee14ec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 shutdown_connections 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 1 -- 172.21.15.90:0/2324552625 wait complete. 2024-04-06T14:21:04.497 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.476+0000 7fa9b681bb80 20 client.18078 trim_cache size 0 max 16384 2024-04-06T14:21:04.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:04 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3710786481' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:04.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 Processor -- start 2024-04-06T14:21:04.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 -- start start 2024-04-06T14:21:04.504 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee154780 0x563eee16ea40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee14f3a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee149d20 con 0x563eee1550a0 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee150010 con 0x563eee154780 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee16f5a0 con 0x563eee1631f0 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9a37fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34524/0 (socket says 172.21.15.90:34524) 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9a37fe700 1 -- 172.21.15.90:0/3685469284 learned_addr learned my addr 172.21.15.90:0/3685469284 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee154780 0x563eee16ea40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.489+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee14f3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.505 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 -- 172.21.15.90:0/3685469284 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 msgr2=0x563eee14f3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee14f3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 -- 172.21.15.90:0/3685469284 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee154780 msgr2=0x563eee16ea40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee154780 0x563eee16ea40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 -- 172.21.15.90:0/3685469284 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563eee06bea0 con 0x563eee1550a0 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 secure :-1 s=READY pgs=910 cs=0 l=1 rev1=1 crypto rx=0x7fa998029db0 tx=0x7fa99801f440 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a70c8f4436f3ee32 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa98affd700 1 -- 172.21.15.90:0/3685469284 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa998004220 con 0x563eee1550a0 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa98affd700 1 -- 172.21.15.90:0/3685469284 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563eee066380 con 0x563eee1550a0 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa98affd700 1 -- 172.21.15.90:0/3685469284 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9980038e0 con 0x563eee1550a0 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3685469284 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 msgr2=0x563eee169be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 secure :-1 s=READY pgs=910 cs=0 l=1 rev1=1 crypto rx=0x7fa998029db0 tx=0x7fa99801f440 comp rx=0 tx=0).stop 2024-04-06T14:21:04.506 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3685469284 shutdown_connections 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee14f3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee154780 0x563eee16ea40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/3685469284 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee1550a0 0x563eee169be0 unknown :-1 s=CLOSED pgs=910 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.490+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3685469284 >> 172.21.15.90:0/3685469284 conn(0x563eee0b36a0 msgr2=0x563eee16e330 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3685469284 shutdown_connections 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- 172.21.15.90:0/3685469284 wait complete. 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- start start 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 Processor -- start 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee167ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee168530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee168a70 con 0x563eee154780 2024-04-06T14:21:04.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee168be0 con 0x563eee1550a0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x563eee168da0 con 0x563eee1631f0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9b681bb80 10 client.0 _ll_register_callbacks cb 0x563eee04b470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9a3fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34526/0 (socket says 172.21.15.90:34526) 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9a3fff700 1 -- 172.21.15.90:0/1619840756 learned_addr learned my addr 172.21.15.90:0/1619840756 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee167ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.491+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee168530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 -- 172.21.15.90:0/1619840756 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 msgr2=0x563eee168530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee168530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 -- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 msgr2=0x563eee167ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee167ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 -- 172.21.15.90:0/1619840756 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x563eeded3b60 con 0x563eee154780 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a8c04700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee168530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:04.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9a3fff700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 secure :-1 s=READY pgs=911 cs=0 l=1 rev1=1 crypto rx=0x7fa99400f740 tx=0x7fa9940b5040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=71fe1cc9fc316d90 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 10 client.0 ms_handle_connect on v2:172.21.15.90:3300/0 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9940b9040 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mon.0 v2:172.21.15.90:3300/0 2 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fa9940b8040 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa99409f0a0 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_get_version_reply(handle=1 version=24) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa99409b270 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 10 client.18084 fetch_fsmap learned FSMap version 24 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,fsmap=24,monmap=4+}) v3 -- 0x563eedf2bcd0 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mon.0 v2:172.21.15.90:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa99409f0a0 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mon.0 v2:172.21.15.90:3300/0 5 ==== fsmap(e 24) v1 ==== 7462+0+0 (secure 0 0 0) 0x7fa9940a31d0 con 0x563eee154780 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 10 client.18084 fetch_fsmap finished waiting for FSMap version 24 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 10 client.18084 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 20 client.18084 populate_metadata read hostname 'smithi090' 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee1c7a00 0x563eee1c9df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:04.509 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 4 client.18084 mds_command: new command op to 24515 tid=0 multi_id=0 [{"prefix": "damage ls"}] 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.492+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 --> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x563eedef4cf0 con 0x563eee1c7a00 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee1c7a00 0x563eee1c9df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9837fe700 10 client.18084 ms_handle_connect on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9a37fe700 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee1c7a00 0x563eee1c9df0 crc :-1 s=READY pgs=261 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fc4f0b306196e276 server_cookie=f622317aa537cb9 in_seq=0 out_seq=0 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9837fe700 1 -- 172.21.15.90:0/1619840756 <== mds.0 v2:172.21.15.105:6838/2253165057 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x563eedef4cf0 con 0x563eee1c7a00 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9837fe700 10 client.18084 handle_command_reply: tid=0 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 client.18084 shutdown 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 msgr2=0x563eee169a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 secure :-1 s=READY pgs=911 cs=0 l=1 rev1=1 crypto rx=0x7fa99400f740 tx=0x7fa9940b5040 comp rx=0 tx=0).stop 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 shutdown_connections 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:6838/2253165057,v1:172.21.15.105:6839/2253165057] conn(0x563eee1c7a00 0x563eee1c9df0 crc :-1 s=READY pgs=261 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.510 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x563eee1631f0 0x563eee168530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x563eee154780 0x563eee169a00 unknown :-1 s=CLOSED pgs=911 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 --2- 172.21.15.90:0/1619840756 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x563eee1550a0 0x563eee167ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9837fe700 0 client.18084 ms_handle_reset on v2:172.21.15.105:6838/2253165057 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 >> 172.21.15.90:0/1619840756 conn(0x563eee0b36a0 msgr2=0x563eee15fc30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 shutdown_connections 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.493+0000 7fa9b681bb80 1 -- 172.21.15.90:0/1619840756 wait complete. 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9b681bb80 20 client.18084 trim_cache size 0 max 16384 2024-04-06T14:21:04.511 INFO:teuthology.orchestra.run.smithi090.stdout:[] 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 msgr2=0x7fa98c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 0x7fa98c0a37d0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7fa9a40685f0 tx=0x7fa998009260 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 msgr2=0x7fa9a41980f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 secure :-1 s=READY pgs=907 cs=0 l=1 rev1=1 crypto rx=0x7fa994005910 tx=0x7fa994009570 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 shutdown_connections 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fa98c0a1310 0x7fa98c0a37d0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa9a4198b70 0x7fa9a41a3170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fa9a40ffe70 0x7fa9a41980f0 unknown :-1 s=CLOSED pgs=907 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 --2- 172.21.15.90:0/2146930548 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fa9a4104f70 0x7fa9a4198630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 >> 172.21.15.90:0/2146930548 conn(0x7fa9a4076d50 msgr2=0x7fa9a40fdf50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 shutdown_connections 2024-04-06T14:21:04.513 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:04.494+0000 7fa9aa667700 1 -- 172.21.15.90:0/2146930548 wait complete. 2024-04-06T14:21:04.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:04 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3710786481' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:05.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:05 smithi105 ceph-mon[46969]: pgmap v491: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 435 KiB/s rd, 16 MiB/s wr, 1.57k op/s 2024-04-06T14:21:05.514 INFO:tasks.fwd_scrub.fs.[cephfs]:end scrubbing fs: cephfs 2024-04-06T14:21:05.514 INFO:tasks.fwd_scrub:done joining 2024-04-06T14:21:05.515 DEBUG:teuthology.run_tasks:Unwinding manager check-counter 2024-04-06T14:21:05.526 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:21:05.537 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:05 smithi090 ceph-mon[43420]: pgmap v491: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 435 KiB/s rd, 16 MiB/s wr, 1.57k op/s 2024-04-06T14:21:05.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:05 smithi134 ceph-mon[47549]: pgmap v491: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 435 KiB/s rd, 16 MiB/s wr, 1.57k op/s 2024-04-06T14:21:05.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- 172.21.15.90:0/3020194653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f0105b80 msgr2=0x7f48f0105f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:05.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 --2- 172.21.15.90:0/3020194653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f0105b80 0x7f48f0105f60 secure :-1 s=READY pgs=912 cs=0 l=1 rev1=1 crypto rx=0x7f48d8009990 tx=0x7f48d8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- 172.21.15.90:0/3020194653 shutdown_connections 2024-04-06T14:21:05.671 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 --2- 172.21.15.90:0/3020194653 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f01004a0 0x7f48f010c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 --2- 172.21.15.90:0/3020194653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f0105b80 0x7f48f0105f60 unknown :-1 s=CLOSED pgs=912 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 --2- 172.21.15.90:0/3020194653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f00ffae0 0x7f48f00fff60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- 172.21.15.90:0/3020194653 >> 172.21.15.90:0/3020194653 conn(0x7f48f00fb5f0 msgr2=0x7f48f00fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:05.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- 172.21.15.90:0/3020194653 shutdown_connections 2024-04-06T14:21:05.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- 172.21.15.90:0/3020194653 wait complete. 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 Processor -- start 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.672+0000 7f48f5839700 1 -- start start 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f01004a0 0x7f48f0197530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f0105b80 0x7f48f019b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f019bf10 con 0x7f48f00ffae0 2024-04-06T14:21:05.674 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f019c080 con 0x7f48f01004a0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f019c240 con 0x7f48f0105b80 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34546/0 (socket says 172.21.15.90:34546) 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 -- 172.21.15.90:0/3302189355 learned_addr learned my addr 172.21.15.90:0/3302189355 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48ee7fc700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f01004a0 0x7f48f0197530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48ef7fe700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f0105b80 0x7f48f019b8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 -- 172.21.15.90:0/3302189355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f0105b80 msgr2=0x7f48f019b8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f0105b80 0x7f48f019b8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 -- 172.21.15.90:0/3302189355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f01004a0 msgr2=0x7f48f0197530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f01004a0 0x7f48f0197530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 -- 172.21.15.90:0/3302189355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48d80095f0 con 0x7f48f00ffae0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48eeffd700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 secure :-1 s=READY pgs=913 cs=0 l=1 rev1=1 crypto rx=0x7f48d80098e0 tx=0x7f48d800f670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48d801d070 con 0x7f48f00ffae0 2024-04-06T14:21:05.675 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f48d8022470 con 0x7f48f00ffae0 2024-04-06T14:21:05.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48d8017650 con 0x7f48f00ffae0 2024-04-06T14:21:05.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.673+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48f019c4c0 con 0x7f48f00ffae0 2024-04-06T14:21:05.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.674+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48f01a3d60 con 0x7f48f00ffae0 2024-04-06T14:21:05.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.674+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f48f01090f0 con 0x7f48f00ffae0 2024-04-06T14:21:05.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.675+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f48d80225e0 con 0x7f48f00ffae0 2024-04-06T14:21:05.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.676+0000 7f48f4837700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 0x7f48dc0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:05.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.676+0000 7f48ee7fc700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 0x7f48dc0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:05.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.676+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f48d80cdcc0 con 0x7f48f00ffae0 2024-04-06T14:21:05.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.677+0000 7f48ee7fc700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 0x7f48dc0a37d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f48f0198540 tx=0x7f48e000b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:05.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.680+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f48d8097180 con 0x7f48f00ffae0 2024-04-06T14:21:05.880 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.881+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f48f01011e0 con 0x7f48f00ffae0 2024-04-06T14:21:05.882 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.882+0000 7f48f4837700 1 -- 172.21.15.90:0/3302189355 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v102) v1 ==== 74+0+23014 (secure 0 0 0) 0x7f48d809a7a0 con 0x7f48f00ffae0 2024-04-06T14:21:05.882 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:05.882 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":102,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:20:13.013889+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"101","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":25,"snap_epoch":101,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":12,"length":1},{"begin":25,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:21:05.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 msgr2=0x7f48dc0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:05.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 0x7f48dc0a37d0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f48f0198540 tx=0x7f48e000b410 comp rx=0 tx=0).stop 2024-04-06T14:21:05.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 msgr2=0x7f48f0196ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:05.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 secure :-1 s=READY pgs=913 cs=0 l=1 rev1=1 crypto rx=0x7f48d80098e0 tx=0x7f48d800f670 comp rx=0 tx=0).stop 2024-04-06T14:21:05.884 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 shutdown_connections 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f48dc0a1310 0x7f48dc0a37d0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f48f0105b80 0x7f48f019b8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f48f00ffae0 0x7f48f0196ff0 unknown :-1 s=CLOSED pgs=913 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 --2- 172.21.15.90:0/3302189355 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f48f01004a0 0x7f48f0197530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 >> 172.21.15.90:0/3302189355 conn(0x7f48f00fb5f0 msgr2=0x7f48f00695a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 shutdown_connections 2024-04-06T14:21:05.885 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:05.885+0000 7f48f5839700 1 -- 172.21.15.90:0/3302189355 wait complete. 2024-04-06T14:21:05.897 INFO:tasks.check_counter.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '101', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 25, 'snap_epoch': 101, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:21:05.897 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/3198345940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e080ffe70 msgr2=0x7f9e08100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/3198345940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e080ffe70 0x7f9e08100250 secure :-1 s=READY pgs=914 cs=0 l=1 rev1=1 crypto rx=0x7f9dfc009990 tx=0x7f9dfc009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/3198345940 shutdown_connections 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/3198345940 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e08104f70 0x7f9e0810caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/3198345940 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e080ffe70 0x7f9e08100250 unknown :-1 s=CLOSED pgs=914 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.066 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/3198345940 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e08100820 0x7f9e08104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/3198345940 >> 172.21.15.90:0/3198345940 conn(0x7f9e08076d50 msgr2=0x7f9e08077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/3198345940 shutdown_connections 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.064+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/3198345940 wait complete. 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 Processor -- start 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 -- start start 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e080ffe70 0x7f9e081971d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e0819bab0 0x7f9e0819bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e0819c530 con 0x7f9e08104f70 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e0819c6a0 con 0x7f9e0819bab0 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0f1ac700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9e0819c860 con 0x7f9e080ffe70 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e07fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:06.067 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e07fff700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34570/0 (socket says 172.21.15.90:34570) 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e07fff700 1 -- 172.21.15.90:0/2471316834 learned_addr learned my addr 172.21.15.90:0/2471316834 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.065+0000 7f9e0d749700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e0819bab0 0x7f9e0819bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 -- 172.21.15.90:0/2471316834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e080ffe70 msgr2=0x7f9e081971d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e080ffe70 0x7f9e081971d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 -- 172.21.15.90:0/2471316834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e0819bab0 msgr2=0x7f9e0819bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e0819bab0 0x7f9e0819bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 -- 172.21.15.90:0/2471316834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9dfc0095f0 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e07fff700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 secure :-1 s=READY pgs=915 cs=0 l=1 rev1=1 crypto rx=0x7f9df400c8f0 tx=0x7f9df400cc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9df4007940 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9e081a3dc0 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9df400f450 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9df4018660 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.066+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9e081a4250 con 0x7f9e08104f70 2024-04-06T14:21:06.068 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.067+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9e081091f0 con 0x7f9e08104f70 2024-04-06T14:21:06.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.068+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f9df400f5c0 con 0x7f9e08104f70 2024-04-06T14:21:06.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.068+0000 7f9e05ffb700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 0x7f9df80a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.068+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(102..102 src has 1..102) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f9df40ccc90 con 0x7f9e08104f70 2024-04-06T14:21:06.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.068+0000 7f9e0cf48700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 0x7f9df80a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:06.069 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.069+0000 7f9e0cf48700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 0x7f9df80a3750 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f9dfc00b5c0 tx=0x7f9dfc005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:06.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.072+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f9df4096150 con 0x7f9e08104f70 2024-04-06T14:21:06.292 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.293+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f9e08101320 con 0x7f9e08104f70 2024-04-06T14:21:06.301 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:21:06.305 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.302+0000 7f9e05ffb700 1 -- 172.21.15.90:0/2471316834 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v103) v1 ==== 93+0+10 (secure 0 0 0) 0x7f9df4099770 con 0x7f9e08104f70 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 msgr2=0x7f9df80a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 0x7f9df80a3750 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f9dfc00b5c0 tx=0x7f9dfc005fb0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 msgr2=0x7f9e08197710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 secure :-1 s=READY pgs=915 cs=0 l=1 rev1=1 crypto rx=0x7f9df400c8f0 tx=0x7f9df400cc00 comp rx=0 tx=0).stop 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 shutdown_connections 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f9df80a1290 0x7f9df80a3750 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9e0819bab0 0x7f9e0819bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.308 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f9e080ffe70 0x7f9e081971d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 --2- 172.21.15.90:0/2471316834 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f9e08104f70 0x7f9e08197710 unknown :-1 s=CLOSED pgs=915 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.306+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 >> 172.21.15.90:0/2471316834 conn(0x7f9e08076d50 msgr2=0x7f9e08068260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:06.309 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.307+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 shutdown_connections 2024-04-06T14:21:06.317 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.318+0000 7f9e0f1ac700 1 -- 172.21.15.90:0/2471316834 wait complete. 2024-04-06T14:21:06.339 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:21:06.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:06 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3302189355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:06.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:06 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3302189355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:06.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.599+0000 7ff6d8692700 1 -- 172.21.15.90:0/960059595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f4b0 msgr2=0x7ff6d0107db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.598 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.599+0000 7ff6d8692700 1 --2- 172.21.15.90:0/960059595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f4b0 0x7ff6d0107db0 secure :-1 s=READY pgs=916 cs=0 l=1 rev1=1 crypto rx=0x7ff6c0009990 tx=0x7ff6c0009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.600+0000 7ff6d8692700 1 -- 172.21.15.90:0/960059595 shutdown_connections 2024-04-06T14:21:06.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.600+0000 7ff6d8692700 1 --2- 172.21.15.90:0/960059595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6d011a590 0x7ff6d011c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.600+0000 7ff6d8692700 1 --2- 172.21.15.90:0/960059595 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f4b0 0x7ff6d0107db0 unknown :-1 s=CLOSED pgs=916 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.600+0000 7ff6d8692700 1 --2- 172.21.15.90:0/960059595 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6d010f920 0x7ff6d01082f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.599 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.600+0000 7ff6d8692700 1 -- 172.21.15.90:0/960059595 >> 172.21.15.90:0/960059595 conn(0x7ff6d006be00 msgr2=0x7ff6d006c210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:06.601 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.601+0000 7ff6d8692700 1 -- 172.21.15.90:0/960059595 shutdown_connections 2024-04-06T14:21:06.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.602+0000 7ff6d8692700 1 -- 172.21.15.90:0/960059595 wait complete. 2024-04-06T14:21:06.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.603+0000 7ff6d8692700 1 Processor -- start 2024-04-06T14:21:06.606 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.604+0000 7ff6d8692700 1 -- start start 2024-04-06T14:21:06.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6d010f4b0 0x7ff6d01a6170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6d011a590 0x7ff6d01aaa50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.607 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6d01ab090 con 0x7ff6d010f920 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6d01ab200 con 0x7ff6d011a590 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d8692700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6d01ab3c0 con 0x7ff6d010f4b0 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34594/0 (socket says 172.21.15.90:34594) 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 -- 172.21.15.90:0/204435009 learned_addr learned my addr 172.21.15.90:0/204435009 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 -- 172.21.15.90:0/204435009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6d010f4b0 msgr2=0x7ff6d01a6170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6d010f4b0 0x7ff6d01a6170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 -- 172.21.15.90:0/204435009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6d011a590 msgr2=0x7ff6d01aaa50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.605+0000 7ff6d5c2d700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6d011a590 0x7ff6d01aaa50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6d5c2d700 1 -- 172.21.15.90:0/204435009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff6c00095f0 con 0x7ff6d010f920 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6d5c2d700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 secure :-1 s=READY pgs=917 cs=0 l=1 rev1=1 crypto rx=0x7ff6cc009fd0 tx=0x7ff6cc00b8b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6cc022640 con 0x7ff6d010f920 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff6cc022c60 con 0x7ff6d010f920 2024-04-06T14:21:06.608 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff6cc021da0 con 0x7ff6d010f920 2024-04-06T14:21:06.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff6d01ab6a0 con 0x7ff6d010f920 2024-04-06T14:21:06.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.606+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff6d010ff50 con 0x7ff6d010f920 2024-04-06T14:21:06.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.608+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff6b40052f0 con 0x7ff6d010f920 2024-04-06T14:21:06.609 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.610+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7ff6cc0227a0 con 0x7ff6d010f920 2024-04-06T14:21:06.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.610+0000 7ff6c77fe700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 0x7ff6bc0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:06.610 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.611+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 8600+0+0 (secure 0 0 0) 0x7ff6cc0dd4f0 con 0x7ff6d010f920 2024-04-06T14:21:06.611 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.612+0000 7ff6d642e700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 0x7ff6bc0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:06.615 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.616+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7ff6cc0a69a0 con 0x7ff6d010f920 2024-04-06T14:21:06.622 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.623+0000 7ff6d642e700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 0x7ff6bc0a3890 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff6c000b5c0 tx=0x7ff6c0000bc0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:06.645 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:06 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3302189355' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:06.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.953+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7ff6b4005b20 con 0x7ff6d010f920 2024-04-06T14:21:06.953 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.954+0000 7ff6c77fe700 1 -- 172.21.15.90:0/204435009 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v103) v1 ==== 104+0+11 (secure 0 0 0) 0x7ff6cc0a9fc0 con 0x7ff6d010f920 2024-04-06T14:21:06.954 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 msgr2=0x7ff6bc0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 0x7ff6bc0a3890 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff6c000b5c0 tx=0x7ff6c0000bc0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 msgr2=0x7ff6d01a66b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 secure :-1 s=READY pgs=917 cs=0 l=1 rev1=1 crypto rx=0x7ff6cc009fd0 tx=0x7ff6cc00b8b0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 shutdown_connections 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7ff6bc0a13d0 0x7ff6bc0a3890 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6d011a590 0x7ff6d01aaa50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7ff6d010f4b0 0x7ff6d01a6170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 --2- 172.21.15.90:0/204435009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7ff6d010f920 0x7ff6d01a66b0 unknown :-1 s=CLOSED pgs=917 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 >> 172.21.15.90:0/204435009 conn(0x7ff6d006be00 msgr2=0x7ff6d0118580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:06.956 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 shutdown_connections 2024-04-06T14:21:06.957 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:06.956+0000 7ff6d8692700 1 -- 172.21.15.90:0/204435009 wait complete. 2024-04-06T14:21:06.968 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- 172.21.15.90:0/3931994783 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c100820 msgr2=0x7f776c104860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 --2- 172.21.15.90:0/3931994783 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c100820 0x7f776c104860 secure :-1 s=READY pgs=918 cs=0 l=1 rev1=1 crypto rx=0x7f7758009990 tx=0x7f7758009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- 172.21.15.90:0/3931994783 shutdown_connections 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 --2- 172.21.15.90:0/3931994783 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c104f90 0x7f776c10cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 --2- 172.21.15.90:0/3931994783 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c0ffe70 0x7f776c100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.096 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 --2- 172.21.15.90:0/3931994783 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c100820 0x7f776c104860 unknown :-1 s=CLOSED pgs=918 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- 172.21.15.90:0/3931994783 >> 172.21.15.90:0/3931994783 conn(0x7f776c076d80 msgr2=0x7f776c077190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- 172.21.15.90:0/3931994783 shutdown_connections 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- 172.21.15.90:0/3931994783 wait complete. 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 Processor -- start 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.094+0000 7f7772c50700 1 -- start start 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c104f90 0x7f776c1966a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c19a9f0 0x7f776c19ae70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f776c19b470 con 0x7f776c0ffe70 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f776c19b5e0 con 0x7f776c104f90 2024-04-06T14:21:07.097 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f776c19b7a0 con 0x7f776c19a9f0 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34600/0 (socket says 172.21.15.90:34600) 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 -- 172.21.15.90:0/3459786841 learned_addr learned my addr 172.21.15.90:0/3459786841 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f776bfff700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c104f90 0x7f776c1966a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77711ed700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c19a9f0 0x7f776c19ae70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 -- 172.21.15.90:0/3459786841 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c19a9f0 msgr2=0x7f776c19ae70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c19a9f0 0x7f776c19ae70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 -- 172.21.15.90:0/3459786841 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c104f90 msgr2=0x7f776c1966a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c104f90 0x7f776c1966a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 -- 172.21.15.90:0/3459786841 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f77580095f0 con 0x7f776c0ffe70 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f77709ec700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7f776000d950 tx=0x7f776000dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:07.098 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7760009940 con 0x7f776c0ffe70 2024-04-06T14:21:07.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7760010460 con 0x7f776c0ffe70 2024-04-06T14:21:07.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f776000f620 con 0x7f776c0ffe70 2024-04-06T14:21:07.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.095+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f776c1a2d00 con 0x7f776c0ffe70 2024-04-06T14:21:07.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.096+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f776c1a3190 con 0x7f776c0ffe70 2024-04-06T14:21:07.099 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.096+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f776c108260 con 0x7f776c0ffe70 2024-04-06T14:21:07.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.097+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f77600108d0 con 0x7f776c0ffe70 2024-04-06T14:21:07.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.098+0000 7f7769ffb700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 0x7f775c0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.098+0000 7f776bfff700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 0x7f775c0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.098+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f77600ccbf0 con 0x7f776c0ffe70 2024-04-06T14:21:07.100 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.099+0000 7f776bfff700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 0x7f775c0a37d0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f776c197660 tx=0x7f7758005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:07.102 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.102+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f7760095f90 con 0x7f776c0ffe70 2024-04-06T14:21:07.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.301+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f776c197410 con 0x7f776c0ffe70 2024-04-06T14:21:07.300 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.301+0000 7f7769ffb700 1 -- 172.21.15.90:0/3459786841 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v103) v1 ==== 100+0+11 (secure 0 0 0) 0x7f77600160a0 con 0x7f776c0ffe70 2024-04-06T14:21:07.301 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:21:07.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 msgr2=0x7f775c0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 0x7f775c0a37d0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f776c197660 tx=0x7f7758005fd0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 msgr2=0x7f776c196160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 secure :-1 s=READY pgs=919 cs=0 l=1 rev1=1 crypto rx=0x7f776000d950 tx=0x7f776000dc60 comp rx=0 tx=0).stop 2024-04-06T14:21:07.303 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 shutdown_connections 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f775c0a1310 0x7f775c0a37d0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f776c19a9f0 0x7f776c19ae70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f776c0ffe70 0x7f776c196160 unknown :-1 s=CLOSED pgs=919 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 --2- 172.21.15.90:0/3459786841 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f776c104f90 0x7f776c1966a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.303+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 >> 172.21.15.90:0/3459786841 conn(0x7f776c076d80 msgr2=0x7f776c0ff3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.304+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 shutdown_connections 2024-04-06T14:21:07.304 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.304+0000 7f7772c50700 1 -- 172.21.15.90:0/3459786841 wait complete. 2024-04-06T14:21:07.315 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json-pretty 2024-04-06T14:21:07.439 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- 172.21.15.90:0/2629410551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 msgr2=0x7f27881021d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2629410551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f27881021d0 secure :-1 s=READY pgs=920 cs=0 l=1 rev1=1 crypto rx=0x7f2778009990 tx=0x7f2778009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- 172.21.15.90:0/2629410551 shutdown_connections 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2629410551 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 0x7f278810d6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2629410551 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f27881021d0 unknown :-1 s=CLOSED pgs=920 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2629410551 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f278810abf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- 172.21.15.90:0/2629410551 >> 172.21.15.90:0/2629410551 conn(0x7f27880fb650 msgr2=0x7f27880fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- 172.21.15.90:0/2629410551 shutdown_connections 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- 172.21.15.90:0/2629410551 wait complete. 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 Processor -- start 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.438+0000 7f278fdf1700 1 -- start start 2024-04-06T14:21:07.440 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f2788197570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 0x7f278819b910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f278819bf50 con 0x7f2788101df0 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f278819c0c0 con 0x7f2788102710 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f278819c280 con 0x7f278810b210 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:34616/0 (socket says 172.21.15.90:34616) 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 -- 172.21.15.90:0/2276685300 learned_addr learned my addr 172.21.15.90:0/2276685300 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278e38e700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 0x7f278819b910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278d38c700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f2788197570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 -- 172.21.15.90:0/2276685300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 msgr2=0x7f278819b910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 0x7f278819b910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 -- 172.21.15.90:0/2276685300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 msgr2=0x7f2788197570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:07.441 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f2788197570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 -- 172.21.15.90:0/2276685300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27780095f0 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278d38c700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f2788197570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278db8d700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 secure :-1 s=READY pgs=921 cs=0 l=1 rev1=1 crypto rx=0x7f2778009fd0 tx=0x7f277800f670 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f277801d070 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2778005510 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2778004b30 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.439+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f278819c500 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.440+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f278819c780 con 0x7f2788101df0 2024-04-06T14:21:07.442 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.440+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f27881062a0 con 0x7f2788101df0 2024-04-06T14:21:07.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.441+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f277800fb90 con 0x7f2788101df0 2024-04-06T14:21:07.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.442+0000 7f277effd700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 0x7f27740a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:07.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.442+0000 7f278d38c700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 0x7f27740a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:07.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.442+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 8600+0+0 (secure 0 0 0) 0x7f27780cd960 con 0x7f2788101df0 2024-04-06T14:21:07.443 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.443+0000 7f278d38c700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 0x7f27740a3750 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2784005950 tx=0x7f27840058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:07.446 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.447+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f2778096df0 con 0x7f2788101df0 2024-04-06T14:21:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:07 smithi105 ceph-mon[46969]: pgmap v492: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 414 KiB/s rd, 13 MiB/s wr, 1.28k op/s 2024-04-06T14:21:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:07 smithi105 ceph-mon[46969]: osdmap e103: 12 total, 12 up, 12 in 2024-04-06T14:21:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:07 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2471316834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:07 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/204435009' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:07 smithi090 ceph-mon[43420]: pgmap v492: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 414 KiB/s rd, 13 MiB/s wr, 1.28k op/s 2024-04-06T14:21:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:07 smithi090 ceph-mon[43420]: osdmap e103: 12 total, 12 up, 12 in 2024-04-06T14:21:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:07 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2471316834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:07 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/204435009' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:07 smithi134 ceph-mon[47549]: pgmap v492: 97 pgs: 97 active+clean; 6.0 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 414 KiB/s rd, 13 MiB/s wr, 1.28k op/s 2024-04-06T14:21:07.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:07 smithi134 ceph-mon[47549]: osdmap e103: 12 total, 12 up, 12 in 2024-04-06T14:21:07.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:07 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2471316834' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.644 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:07 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/204435009' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:07.699 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.700+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json-pretty"} v 0) v1 -- 0x7f278819c780 con 0x7f2788101df0 2024-04-06T14:21:07.707 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.708+0000 7f277effd700 1 -- 172.21.15.90:0/2276685300 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json-pretty"}]=0 v25) v1 ==== 81+0+733941 (secure 0 0 0) 0x7f278819c780 con 0x7f2788101df0 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "epoch": 25, 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "active_gid": 15288, 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "active_name": "x", 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "active_addrs": { 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:07.717 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:6800", 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 2273339784 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v1", 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:6801", 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 2273339784 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "active_addr": "172.21.15.90:6801/2273339784", 2024-04-06T14:21:07.718 INFO:teuthology.orchestra.run.smithi090.stdout: "active_change": "2024-04-06T14:05:23.551562+0000", 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "active_mgr_features": 4540138322906710015, 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "available": true, 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "standbys": [ 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "gid": 24895, 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "z", 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "mgr_features": 4540138322906710015, 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "available_modules": [ 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "alerts", 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.719 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How frequently to reexamine health status", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.720 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.721 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.722 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.723 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.724 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.725 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_destination": { 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_destination", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email address to send alerts to", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.726 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_from_name": { 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_from_name", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "Ceph", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email From: name", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.727 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_host": { 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_host", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP server", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.728 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_password": { 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_password", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Password to authenticate with", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_port": { 2024-04-06T14:21:07.729 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_port", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "465", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP port", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_sender": { 2024-04-06T14:21:07.730 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_sender", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP envelope sender", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_ssl": { 2024-04-06T14:21:07.731 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_ssl", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_user": { 2024-04-06T14:21:07.732 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_user", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "User to authenticate as", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.733 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "balancer", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "active": { 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "active", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.734 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically balance PGs across cluster", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_time": { 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_time", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0000", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.735 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "beginning time of day to automatically balance", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_weekday": { 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_weekday", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:07.736 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_max_iterations": { 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_max_iterations", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "25", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:07.737 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "250", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_metrics": { 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_metrics", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "pgs,objects,bytes", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.738 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_step": { 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_step", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.5", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0.001", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "0.999", 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.739 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "aggressiveness of optimization", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "end_time": { 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_time", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2359", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ending time of day to automatically balance", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:07.740 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "end_weekday": { 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_weekday", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.741 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.742 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.743 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.744 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.745 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.746 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "min_score": { 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_score", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.747 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "upmap", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "crush-compat", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "none", 2024-04-06T14:21:07.748 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap" 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Balancer mode", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_ids": { 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_ids", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.749 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.750 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_deviation": { 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_deviation", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.751 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_optimizations": { 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_optimizations", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.752 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cephadm", 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_down_multiplier": { 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_down_multiplier", 2024-04-06T14:21:07.753 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_refresh_rate": { 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_refresh_rate", 2024-04-06T14:21:07.754 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "20", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_starting_port": { 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_starting_port", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.755 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4721", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_ptrace": { 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_ptrace", 2024-04-06T14:21:07.756 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_interval": { 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_interval", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.757 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_memory_target_ratio": { 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_memory_target_ratio", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:07.758 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.7", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "cgroups_split": { 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cgroups_split", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.759 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "config_checks_enabled": { 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_checks_enabled", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.760 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "config_dashboard": { 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_dashboard", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.761 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_alertmanager": { 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_alertmanager", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.762 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_base": { 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_base", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.763 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Container image name, without the tag", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_elasticsearch": { 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_elasticsearch", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.764 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "elasticsearch container image", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_grafana": { 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_grafana", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.765 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_haproxy": { 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_haproxy", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-06T14:21:07.766 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "HAproxy container image", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_agent": { 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_agent", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.767 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger agent container image", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_collector": { 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_collector", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-06T14:21:07.768 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger collector container image", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_query": { 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_query", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.769 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger query container image", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_keepalived": { 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_keepalived", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.770 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Keepalived container image", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_loki": { 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_loki", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.771 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Loki container image", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_node_exporter": { 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_node_exporter", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.772 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_nvmeof": { 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_nvmeof", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.773 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Nvme-of container image", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_prometheus": { 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_prometheus", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.774 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_promtail": { 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_promtail", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.775 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Promtail container image", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_snmp_gateway": { 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_snmp_gateway", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.776 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SNMP Gateway container image", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "container_init": { 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_init", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.777 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Run podman/docker with `--init`", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "daemon_cache_timeout": { 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "daemon_cache_timeout", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.778 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "default_cephadm_command_timeout": { 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_cephadm_command_timeout", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "900", 2024-04-06T14:21:07.779 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "default_registry": { 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_registry", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.780 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "device_cache_timeout": { 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_cache_timeout", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1800", 2024-04-06T14:21:07.781 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache device inventory", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "device_enhanced_scan": { 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_enhanced_scan", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.782 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "facts_cache_timeout": { 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "facts_cache_timeout", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:07.783 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache host facts data", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "host_check_interval": { 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "host_check_interval", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:07.784 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to perform a host check", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "hw_monitoring": { 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hw_monitoring", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.785 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "inventory_list_all": { 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "inventory_list_all", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.786 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.787 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.788 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "log_refresh_metadata": { 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_refresh_metadata", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.789 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.790 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.791 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.792 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.793 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "*", 2024-04-06T14:21:07.794 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "max_count_per_host": { 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_count_per_host", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.795 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of daemons per service per host", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "max_osd_draining_count": { 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_osd_draining_count", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.796 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "migration_current": { 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "migration_current", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.797 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "internal - do not modify", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.798 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "root", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm-package", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "root" 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "mode for remote execution of cephadm", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "oob_default_addr": { 2024-04-06T14:21:07.799 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "oob_default_addr", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "169.254.1.1", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "prometheus_alerts_path": { 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus_alerts_path", 2024-04-06T14:21:07.800 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_insecure": { 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_insecure", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.801 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_password": { 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_password", 2024-04-06T14:21:07.802 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_url": { 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_url", 2024-04-06T14:21:07.803 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_username": { 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_username", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.804 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.805 INFO:teuthology.orchestra.run.smithi090.stdout: "secure_monitoring_stack": { 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "secure_monitoring_stack", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.806 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "service_discovery_port": { 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "service_discovery_port", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8765", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "cephadm service discovery port", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.807 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "ssh_config_file": { 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssh_config_file", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.808 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "use_agent": { 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_agent", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.809 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "use_repo_digest": { 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_repo_digest", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.810 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_failed_host_check": { 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_failed_host_check", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if the host check fails", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.811 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_daemons": { 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_daemons", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-06T14:21:07.812 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_hosts": { 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_hosts", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.813 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crash", 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.814 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.815 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.816 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.817 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.818 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "retain_interval": { 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retain_interval", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.819 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "31536000", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_recent_interval": { 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_recent_interval", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.820 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1209600", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.821 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dashboard", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.822 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_HOST": { 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.823 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.824 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_ENABLED": { 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_ENABLED", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.825 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.826 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "ENABLE_BROWSABLE_API": { 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.827 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.828 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.829 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-06T14:21:07.830 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-06T14:21:07.831 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.832 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.833 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-06T14:21:07.834 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.835 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-06T14:21:07.836 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_PASSWORD": { 2024-04-06T14:21:07.837 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.838 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_URL": { 2024-04-06T14:21:07.839 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_URL", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_USERNAME": { 2024-04-06T14:21:07.840 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-06T14:21:07.841 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-06T14:21:07.842 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-06T14:21:07.843 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.844 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_HOST": { 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.845 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-06T14:21:07.846 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.847 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.848 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.849 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.850 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.851 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.852 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.853 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.854 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_ENABLED": { 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.855 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.856 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.857 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.858 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "45", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.859 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ACCESS_KEY": { 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.860 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.861 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SECRET_KEY": { 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.862 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SSL_VERIFY": { 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.863 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "UNSAFE_TLS_v1_2": { 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "UNSAFE_TLS_v1_2", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.864 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.865 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.866 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.867 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "cross_origin_url": { 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cross_origin_url", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.868 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "crt_file": { 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crt_file", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.869 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "debug": { 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "debug", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.870 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable/disable debug options", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "jwt_token_ttl": { 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "jwt_token_ttl", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "28800", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.871 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.872 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.873 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.874 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.875 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.876 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.877 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "motd": { 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "motd", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.878 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "The message of the day", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect_resolve_ip_addr": { 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "redirect_resolve_ip_addr", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.879 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.880 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.881 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8080", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.882 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl_server_port": { 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl_server_port", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.883 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8443", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.884 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "redirect", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect" 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.885 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.886 INFO:teuthology.orchestra.run.smithi090.stdout: "url_prefix": { 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url_prefix", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.887 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "devicehealth", 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_monitoring": { 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_monitoring", 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.888 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "monitor device health metrics", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.889 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.890 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.891 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.892 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.893 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.894 INFO:teuthology.orchestra.run.smithi090.stdout: "mark_out_threshold": { 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mark_out_threshold", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2419200", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_name": { 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_name", 2024-04-06T14:21:07.895 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "device_health_metrics", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "retention_period": { 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retention_period", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.896 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15552000", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain device health metrics", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_frequency": { 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_frequency", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:07.897 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "self_heal": { 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "self_heal", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.898 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.899 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and check device health", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_threshold": { 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_threshold", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7257600", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.900 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "diskprediction_local", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.901 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.902 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.903 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.904 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.905 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "predict_interval": { 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predict_interval", 2024-04-06T14:21:07.906 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "predictor_model": { 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predictor_model", 2024-04-06T14:21:07.907 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "prophetstor", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:07.908 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.909 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "influx", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": false, 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "influxdb python module not found", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "batch_size": { 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "batch_size", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5000", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.910 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "database": { 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "database", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "ceph", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-06T14:21:07.911 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "hostname": { 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hostname", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server hostname", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.912 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "30", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "5", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.913 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.914 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.915 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.916 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.917 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.918 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "password": { 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "password", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.919 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "password of InfluxDB server user", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "port": { 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "port", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8086", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server port", 2024-04-06T14:21:07.920 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "false", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.921 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "threads": { 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threads", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "32", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.922 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "username": { 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "username", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "username of InfluxDB server user", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.923 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "verify_ssl": { 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "verify_ssl", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "true", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.924 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "insights", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.925 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.926 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.927 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.928 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.929 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "iostat", 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.930 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.931 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.932 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.933 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.934 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "k8sevents", 2024-04-06T14:21:07.935 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph_event_retention_days": { 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ceph_event_retention_days", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.936 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "config_check_secs": { 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_check_secs", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "10", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.937 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.938 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.939 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.940 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.941 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.942 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "localpool", 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "failure_domain": { 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "failure_domain", 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.943 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "host", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "failure domain for any created local pool", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.944 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.945 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.946 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.947 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.948 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "min_size": { 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_size", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.949 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default min_size for any created local pool", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "num_rep": { 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "num_rep", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.950 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default replica count for any created local pool", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_num": { 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_num", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "128", 2024-04-06T14:21:07.951 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default pg_num for any created local pool", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "prefix": { 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prefix", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.952 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name prefix for any created local pool", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "subtree": { 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "subtree", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "rack", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.953 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mds_autoscaler", 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.954 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.955 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.956 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.957 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.958 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.959 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirroring", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.960 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.961 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.962 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.963 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.964 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "nfs", 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.965 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.966 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.967 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.968 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.969 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.970 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.971 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "fail_fs": { 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "fail_fs", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.972 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.973 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.974 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.975 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.976 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.977 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator": { 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "rook", 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "test_orchestrator" 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.978 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Orchestrator backend", 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_perf_query", 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.979 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.980 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.981 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.982 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.983 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.984 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_support", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.985 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.986 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.987 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.988 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.989 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_autoscaler", 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.990 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.991 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.992 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.993 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:07.994 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:07.995 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "threshold": { 2024-04-06T14:21:07.996 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threshold", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1.0", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "scaling threshold", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:07.997 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "progress", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_pg_recovery_event": { 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_pg_recovery_event", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:07.998 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:07.999 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.000 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.001 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.002 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.003 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.004 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "max_completed_events": { 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_completed_events", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.005 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "50", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "number of past completed events to remember", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.006 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long the module is going to sleep", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus", 2024-04-06T14:21:08.007 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "cache": { 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cache", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.008 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "exclude_perf_counters": { 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "exclude_perf_counters", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-06T14:21:08.009 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.010 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.011 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.012 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.013 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.014 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools": { 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.015 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "300", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.016 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_interval": { 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_interval", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15.0", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.017 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.018 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "9283", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.019 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "stale_cache_strategy": { 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stale_cache_strategy", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "log", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.020 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "default", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "default", 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: "error" 2024-04-06T14:21:08.021 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:08.022 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_support", 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.023 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.024 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.025 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.026 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.027 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_snap_create": { 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_snap_create", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.028 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "mirror_snapshot_schedule": { 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirror_snapshot_schedule", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.029 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "trash_purge_schedule": { 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "trash_purge_schedule", 2024-04-06T14:21:08.030 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.031 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "restful", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_auth": { 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_auth", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.032 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.033 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.034 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.035 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.036 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.037 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.038 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.039 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.040 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rgw", 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.041 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.042 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.043 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.044 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.045 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.046 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rook", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.047 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.048 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.049 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.050 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.051 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "storage_class": { 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "storage_class", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "local", 2024-04-06T14:21:08.052 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "selftest", 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.053 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.054 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.055 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.056 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.057 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.058 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "roption1": { 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption1", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.059 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "roption2": { 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption2", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "xyz", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.060 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption1": { 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption1", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.061 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption2": { 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption2", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.062 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption3": { 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption3", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.063 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption4": { 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption4", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.064 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption5": { 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption5", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.065 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption6": { 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption6", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.066 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption7": { 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption7", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "42", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.067 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "testkey": { 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testkey", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.068 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "testlkey": { 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testlkey", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.069 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "testnewline": { 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testnewline", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.070 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snap_schedule", 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_m_granularity": { 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_m_granularity", 2024-04-06T14:21:08.071 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow minute scheduled snapshots", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "dump_on_update": { 2024-04-06T14:21:08.072 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dump_on_update", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "dump database to debug log on update", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.073 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.074 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.075 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.076 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.077 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.078 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stats", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.079 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.080 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.081 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.082 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.083 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "status", 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.084 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.085 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.086 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.087 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.088 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.089 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telegraf", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "address": { 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "address", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.090 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.091 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.092 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.093 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.094 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.095 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.096 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telemetry", 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.097 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_basic": { 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_basic", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.098 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_crash": { 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_crash", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.099 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_device": { 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_device", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.100 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_ident": { 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_ident", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.101 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_perf": { 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_perf", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.102 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "contact": { 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "contact", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.103 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "description": { 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "description", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.104 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "device_url": { 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_url", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.105 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.106 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "24", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "8", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.107 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "last_opt_revision": { 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "last_opt_revision", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.108 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard": { 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.109 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard_description": { 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard_description", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.110 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.111 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.112 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.113 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.114 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.115 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "organization": { 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "organization", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.116 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "proxy": { 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "proxy", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.117 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "url": { 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-06T14:21:08.118 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "test_orchestrator", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.119 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.120 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.121 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.122 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.123 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.124 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "volumes", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.125 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.126 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.127 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.128 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.129 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_clones": { 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_clones", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-06T14:21:08.130 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_delay": { 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_delay", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.131 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_no_wait": { 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_no_wait", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.132 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix", 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "discovery_interval": { 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "discovery_interval", 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.133 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "100", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "identifier": { 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "identifier", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.134 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.135 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.136 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.137 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.138 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.139 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.140 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_host": { 2024-04-06T14:21:08.141 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_host", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_port": { 2024-04-06T14:21:08.142 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_port", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10051", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.143 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_sender": { 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_sender", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.144 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "gid": 24947, 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "y", 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "mgr_features": 4540138322906710015, 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "available_modules": [ 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "alerts", 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.145 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How frequently to reexamine health status", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.146 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.147 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.148 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.149 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.150 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_destination": { 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_destination", 2024-04-06T14:21:08.151 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email address to send alerts to", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_from_name": { 2024-04-06T14:21:08.152 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_from_name", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "Ceph", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email From: name", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_host": { 2024-04-06T14:21:08.153 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_host", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP server", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_password": { 2024-04-06T14:21:08.154 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_password", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Password to authenticate with", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_port": { 2024-04-06T14:21:08.155 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_port", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "465", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP port", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.156 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_sender": { 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_sender", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP envelope sender", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_ssl": { 2024-04-06T14:21:08.157 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_ssl", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_user": { 2024-04-06T14:21:08.158 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_user", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "User to authenticate as", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.159 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "balancer", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "active": { 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "active", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.160 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically balance PGs across cluster", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_time": { 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_time", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0000", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.161 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "beginning time of day to automatically balance", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_weekday": { 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_weekday", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:08.162 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_max_iterations": { 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_max_iterations", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "25", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "250", 2024-04-06T14:21:08.163 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_metrics": { 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_metrics", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "pgs,objects,bytes", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.164 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_step": { 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_step", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.5", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0.001", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "0.999", 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.165 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "aggressiveness of optimization", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "end_time": { 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_time", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2359", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.166 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ending time of day to automatically balance", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "end_weekday": { 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_weekday", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:08.167 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.168 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.169 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.170 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.171 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.172 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "min_score": { 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_score", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.173 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "upmap", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "crush-compat", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "none", 2024-04-06T14:21:08.174 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap" 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Balancer mode", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_ids": { 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_ids", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.175 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.176 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_deviation": { 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_deviation", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.177 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_optimizations": { 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_optimizations", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.178 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cephadm", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.179 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_down_multiplier": { 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_down_multiplier", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.180 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_refresh_rate": { 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_refresh_rate", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "20", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.181 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_starting_port": { 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_starting_port", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4721", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.182 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_ptrace": { 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_ptrace", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.183 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_interval": { 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_interval", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.184 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_memory_target_ratio": { 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_memory_target_ratio", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.7", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.185 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "cgroups_split": { 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cgroups_split", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.186 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "config_checks_enabled": { 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_checks_enabled", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.187 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "config_dashboard": { 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_dashboard", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.188 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_alertmanager": { 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_alertmanager", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.189 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_base": { 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_base", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Container image name, without the tag", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.190 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_elasticsearch": { 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_elasticsearch", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "elasticsearch container image", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.191 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_grafana": { 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_grafana", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.192 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_haproxy": { 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_haproxy", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "HAproxy container image", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.193 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_agent": { 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_agent", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger agent container image", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.194 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_collector": { 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_collector", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger collector container image", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.195 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_query": { 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_query", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger query container image", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.196 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_keepalived": { 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_keepalived", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Keepalived container image", 2024-04-06T14:21:08.197 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_loki": { 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_loki", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Loki container image", 2024-04-06T14:21:08.198 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_node_exporter": { 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_node_exporter", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.199 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_nvmeof": { 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_nvmeof", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.200 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Nvme-of container image", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_prometheus": { 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_prometheus", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.201 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_promtail": { 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_promtail", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.202 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Promtail container image", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_snmp_gateway": { 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_snmp_gateway", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.203 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SNMP Gateway container image", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "container_init": { 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_init", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.204 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Run podman/docker with `--init`", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "daemon_cache_timeout": { 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "daemon_cache_timeout", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.205 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "default_cephadm_command_timeout": { 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_cephadm_command_timeout", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "900", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.206 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "default_registry": { 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_registry", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.207 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "device_cache_timeout": { 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_cache_timeout", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1800", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.208 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache device inventory", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "device_enhanced_scan": { 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_enhanced_scan", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.209 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "facts_cache_timeout": { 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "facts_cache_timeout", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.210 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache host facts data", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "host_check_interval": { 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "host_check_interval", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.211 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to perform a host check", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "hw_monitoring": { 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hw_monitoring", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.212 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "inventory_list_all": { 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "inventory_list_all", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.213 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.214 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.215 INFO:teuthology.orchestra.run.smithi090.stdout: "log_refresh_metadata": { 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_refresh_metadata", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.216 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.217 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.218 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.219 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.220 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "*", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.221 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "max_count_per_host": { 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_count_per_host", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.222 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of daemons per service per host", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "max_osd_draining_count": { 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_osd_draining_count", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.223 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "migration_current": { 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "migration_current", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.224 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "internal - do not modify", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "root", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.225 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm-package", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "root" 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "mode for remote execution of cephadm", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "oob_default_addr": { 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "oob_default_addr", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "169.254.1.1", 2024-04-06T14:21:08.226 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "prometheus_alerts_path": { 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus_alerts_path", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-06T14:21:08.227 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_insecure": { 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_insecure", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.228 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_password": { 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_password", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.229 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_url": { 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_url", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.230 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_username": { 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_username", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.231 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "secure_monitoring_stack": { 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "secure_monitoring_stack", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.232 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "service_discovery_port": { 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "service_discovery_port", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8765", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.233 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "cephadm service discovery port", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "ssh_config_file": { 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssh_config_file", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.234 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "use_agent": { 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_agent", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.235 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "use_repo_digest": { 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_repo_digest", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.236 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_failed_host_check": { 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_failed_host_check", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.237 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if the host check fails", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_daemons": { 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_daemons", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.238 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_hosts": { 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_hosts", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.239 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crash", 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.240 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.241 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.242 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.243 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.244 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "retain_interval": { 2024-04-06T14:21:08.245 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retain_interval", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "31536000", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.246 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_recent_interval": { 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_recent_interval", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1209600", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.247 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dashboard", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.248 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_HOST": { 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.249 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.250 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_ENABLED": { 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_ENABLED", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.251 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.252 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "ENABLE_BROWSABLE_API": { 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.253 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.254 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.255 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.256 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.257 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.258 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.259 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.260 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.261 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_PASSWORD": { 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.262 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.263 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_URL": { 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_URL", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.264 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_USERNAME": { 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.265 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.266 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.267 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.268 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.269 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_HOST": { 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.270 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.271 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.272 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.273 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.274 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.275 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.276 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.277 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.278 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_ENABLED": { 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.279 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.280 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.281 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.282 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.283 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "45", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ACCESS_KEY": { 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.284 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-06T14:21:08.285 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SECRET_KEY": { 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-06T14:21:08.286 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SSL_VERIFY": { 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-06T14:21:08.287 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "UNSAFE_TLS_v1_2": { 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "UNSAFE_TLS_v1_2", 2024-04-06T14:21:08.288 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-06T14:21:08.289 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-06T14:21:08.290 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.291 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.292 INFO:teuthology.orchestra.run.smithi090.stdout: "cross_origin_url": { 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cross_origin_url", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.293 INFO:teuthology.orchestra.run.smithi090.stdout: "crt_file": { 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crt_file", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.294 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "debug": { 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "debug", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable/disable debug options", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.295 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "jwt_token_ttl": { 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "jwt_token_ttl", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "28800", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.296 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.297 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.298 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.299 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.300 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.301 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.302 INFO:teuthology.orchestra.run.smithi090.stdout: "motd": { 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "motd", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "The message of the day", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.303 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect_resolve_ip_addr": { 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "redirect_resolve_ip_addr", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.304 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.305 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8080", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.306 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.307 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl_server_port": { 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl_server_port", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8443", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.308 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "redirect", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect" 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.309 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.310 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "url_prefix": { 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url_prefix", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.311 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "devicehealth", 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_monitoring": { 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_monitoring", 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.312 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "monitor device health metrics", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.313 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.314 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.315 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.316 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.317 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "mark_out_threshold": { 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mark_out_threshold", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.318 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2419200", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_name": { 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_name", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.319 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "device_health_metrics", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "retention_period": { 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retention_period", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.320 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15552000", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain device health metrics", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_frequency": { 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_frequency", 2024-04-06T14:21:08.321 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "self_heal": { 2024-04-06T14:21:08.322 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "self_heal", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.323 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and check device health", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_threshold": { 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_threshold", 2024-04-06T14:21:08.324 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7257600", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.325 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "diskprediction_local", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.326 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.327 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.328 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.329 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.330 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "predict_interval": { 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predict_interval", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.331 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "predictor_model": { 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predictor_model", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "prophetstor", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.332 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.333 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "influx", 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": false, 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "influxdb python module not found", 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.334 INFO:teuthology.orchestra.run.smithi090.stdout: "batch_size": { 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "batch_size", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5000", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.335 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "database": { 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "database", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "ceph", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.336 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "hostname": { 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hostname", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server hostname", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.337 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "30", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "5", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.338 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.339 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.340 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.341 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.342 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.343 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "password": { 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "password", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.344 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "password of InfluxDB server user", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "port": { 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "port", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8086", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.345 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server port", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "false", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.346 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "threads": { 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threads", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "32", 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.347 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "username": { 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "username", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.348 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "username of InfluxDB server user", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "verify_ssl": { 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "verify_ssl", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "true", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.349 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "insights", 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.350 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.351 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.352 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.353 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.354 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.355 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "iostat", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.356 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.357 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.358 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.359 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.360 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "k8sevents", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.361 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph_event_retention_days": { 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ceph_event_retention_days", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.362 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "config_check_secs": { 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_check_secs", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "10", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.363 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.364 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.365 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.366 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.367 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.368 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "localpool", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "failure_domain": { 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "failure_domain", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "host", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.369 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "failure domain for any created local pool", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.370 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.371 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.372 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.373 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.374 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "min_size": { 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_size", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.375 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default min_size for any created local pool", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "num_rep": { 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "num_rep", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3", 2024-04-06T14:21:08.376 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default replica count for any created local pool", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_num": { 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_num", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "128", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.377 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default pg_num for any created local pool", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "prefix": { 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prefix", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.378 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name prefix for any created local pool", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "subtree": { 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "subtree", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.379 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "rack", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mds_autoscaler", 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.380 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.381 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.382 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.383 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.384 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.385 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirroring", 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.386 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.387 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.388 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.389 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.390 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.391 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "nfs", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.392 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.393 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.394 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.395 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.396 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.397 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "fail_fs": { 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "fail_fs", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.398 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.399 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.400 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.401 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.402 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.403 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator": { 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.404 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "rook", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "test_orchestrator" 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Orchestrator backend", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_perf_query", 2024-04-06T14:21:08.405 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.406 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.407 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.408 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.409 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.410 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_support", 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.411 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.412 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.413 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.414 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.417 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.417 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.417 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.417 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.418 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_autoscaler", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.419 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.420 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.421 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.422 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.423 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.424 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.425 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "threshold": { 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threshold", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1.0", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "scaling threshold", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-06T14:21:08.426 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "progress", 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_pg_recovery_event": { 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_pg_recovery_event", 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.427 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.428 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.429 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.430 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.431 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.432 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.433 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "max_completed_events": { 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_completed_events", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.434 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "50", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "number of past completed events to remember", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.435 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long the module is going to sleep", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus", 2024-04-06T14:21:08.436 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "cache": { 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cache", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.437 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "exclude_perf_counters": { 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "exclude_perf_counters", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.438 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.439 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.440 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.441 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.442 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.443 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools": { 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.444 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "300", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.445 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_interval": { 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_interval", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15.0", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.446 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.447 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "9283", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.448 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "stale_cache_strategy": { 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stale_cache_strategy", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "log", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.449 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "default", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "default", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "error" 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.450 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.451 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_support", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.452 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.453 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.454 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.455 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.456 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_snap_create": { 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_snap_create", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.457 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "mirror_snapshot_schedule": { 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirror_snapshot_schedule", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.458 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "trash_purge_schedule": { 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "trash_purge_schedule", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.459 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "restful", 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.460 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_auth": { 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_auth", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.461 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.462 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.463 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.464 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.465 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.466 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.467 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.468 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.469 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rgw", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.470 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.471 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.472 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.473 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.474 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rook", 2024-04-06T14:21:08.475 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.476 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.477 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.478 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.479 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.480 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "storage_class": { 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "storage_class", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "local", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.481 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "selftest", 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.482 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.483 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.484 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.485 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.486 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "roption1": { 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption1", 2024-04-06T14:21:08.487 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "roption2": { 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption2", 2024-04-06T14:21:08.488 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "xyz", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption1": { 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption1", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.489 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption2": { 2024-04-06T14:21:08.490 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption2", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.491 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption3": { 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption3", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption4": { 2024-04-06T14:21:08.492 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption4", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption5": { 2024-04-06T14:21:08.493 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption5", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption6": { 2024-04-06T14:21:08.494 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption6", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption7": { 2024-04-06T14:21:08.495 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption7", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "42", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "testkey": { 2024-04-06T14:21:08.496 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testkey", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "testlkey": { 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testlkey", 2024-04-06T14:21:08.497 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "testnewline": { 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testnewline", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.498 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.499 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snap_schedule", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_m_granularity": { 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_m_granularity", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow minute scheduled snapshots", 2024-04-06T14:21:08.500 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "dump_on_update": { 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dump_on_update", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.501 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "dump database to debug log on update", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.502 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.503 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.504 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.505 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.506 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stats", 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.507 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.508 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.509 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.510 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.511 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.512 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "status", 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.513 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.514 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.515 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.516 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.517 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telegraf", 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.518 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "address": { 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "address", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.519 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.520 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.521 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.522 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.523 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.524 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.525 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telemetry", 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_basic": { 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_basic", 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.526 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_crash": { 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_crash", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.527 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_device": { 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_device", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.528 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_ident": { 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_ident", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.529 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_perf": { 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_perf", 2024-04-06T14:21:08.530 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "contact": { 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "contact", 2024-04-06T14:21:08.531 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "description": { 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "description", 2024-04-06T14:21:08.532 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "device_url": { 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_url", 2024-04-06T14:21:08.533 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:08.534 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.535 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "24", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "8", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "last_opt_revision": { 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "last_opt_revision", 2024-04-06T14:21:08.536 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard": { 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard", 2024-04-06T14:21:08.537 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard_description": { 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard_description", 2024-04-06T14:21:08.538 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.539 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.540 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.541 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.542 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.543 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "organization": { 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "organization", 2024-04-06T14:21:08.544 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "proxy": { 2024-04-06T14:21:08.545 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "proxy", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.546 INFO:teuthology.orchestra.run.smithi090.stdout: "url": { 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.547 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "test_orchestrator", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.548 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.549 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.550 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.551 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.552 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "volumes", 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.553 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.554 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.555 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.556 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.557 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_clones": { 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_clones", 2024-04-06T14:21:08.558 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_delay": { 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_delay", 2024-04-06T14:21:08.559 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_no_wait": { 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_no_wait", 2024-04-06T14:21:08.560 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.561 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "discovery_interval": { 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "discovery_interval", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "100", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.562 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "identifier": { 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "identifier", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.563 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.564 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.565 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.566 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.567 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.568 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.569 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_host": { 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_host", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.570 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_port": { 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_port", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.571 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10051", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_sender": { 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_sender", 2024-04-06T14:21:08.572 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.573 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "modules": [ 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm", 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "dashboard", 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "iostat", 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "nfs", 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "restful", 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "snap_schedule" 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "available_modules": [ 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.574 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "alerts", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How frequently to reexamine health status", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.575 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.576 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.577 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.578 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.579 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.580 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_destination": { 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_destination", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email address to send alerts to", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.581 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_from_name": { 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_from_name", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "Ceph", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Email From: name", 2024-04-06T14:21:08.582 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_host": { 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_host", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP server", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.583 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_password": { 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_password", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Password to authenticate with", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.584 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_port": { 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_port", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "465", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP port", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.585 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_sender": { 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_sender", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SMTP envelope sender", 2024-04-06T14:21:08.586 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_ssl": { 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_ssl", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use SSL to connect to SMTP server", 2024-04-06T14:21:08.587 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "smtp_user": { 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "smtp_user", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "User to authenticate as", 2024-04-06T14:21:08.588 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "balancer", 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "active": { 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "active", 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.589 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically balance PGs across cluster", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_time": { 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_time", 2024-04-06T14:21:08.590 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0000", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "beginning time of day to automatically balance", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "begin_weekday": { 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "begin_weekday", 2024-04-06T14:21:08.591 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_max_iterations": { 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_max_iterations", 2024-04-06T14:21:08.592 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "25", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "250", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum number of iterations to attempt optimization", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_metrics": { 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_metrics", 2024-04-06T14:21:08.593 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "pgs,objects,bytes", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "metrics with which to calculate OSD utilization", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "crush_compat_step": { 2024-04-06T14:21:08.594 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crush_compat_step", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.5", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0.001", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "0.999", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "aggressiveness of optimization", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "end_time": { 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_time", 2024-04-06T14:21:08.595 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2359", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ending time of day to automatically balance", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "This is a time of day in the format HHMM.", 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "end_weekday": { 2024-04-06T14:21:08.596 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "end_weekday", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "0", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "6", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.597 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.598 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.599 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.600 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.601 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.602 INFO:teuthology.orchestra.run.smithi090.stdout: "min_score": { 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_score", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "minimum score, below which no optimization is attempted", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.603 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "upmap", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "crush-compat", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "none", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap" 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Balancer mode", 2024-04-06T14:21:08.604 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_ids": { 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_ids", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "pools which the automatic balancing will be limited to", 2024-04-06T14:21:08.605 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and attempt optimization", 2024-04-06T14:21:08.606 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_deviation": { 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_deviation", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.607 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "deviation below which no optimization is attempted", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "upmap_max_optimizations": { 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "upmap_max_optimizations", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.608 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "maximum upmap optimizations to make per attempt", 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cephadm", 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_down_multiplier": { 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_down_multiplier", 2024-04-06T14:21:08.609 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.610 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_refresh_rate": { 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_refresh_rate", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "20", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.611 INFO:teuthology.orchestra.run.smithi090.stdout: "agent_starting_port": { 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "agent_starting_port", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4721", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.612 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_ptrace": { 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_ptrace", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.613 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_interval": { 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_interval", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to autotune daemon memory", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.614 INFO:teuthology.orchestra.run.smithi090.stdout: "autotune_memory_target_ratio": { 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "autotune_memory_target_ratio", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0.7", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.615 INFO:teuthology.orchestra.run.smithi090.stdout: "cgroups_split": { 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cgroups_split", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "config_checks_enabled": { 2024-04-06T14:21:08.616 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_checks_enabled", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "config_dashboard": { 2024-04-06T14:21:08.617 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_dashboard", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_alertmanager": { 2024-04-06T14:21:08.618 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_alertmanager", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_base": { 2024-04-06T14:21:08.619 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_base", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Container image name, without the tag", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_elasticsearch": { 2024-04-06T14:21:08.620 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_elasticsearch", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "elasticsearch container image", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.621 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_grafana": { 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_grafana", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.622 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_haproxy": { 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_haproxy", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "HAproxy container image", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.623 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_agent": { 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_agent", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger agent container image", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.624 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_collector": { 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_collector", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger collector container image", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.625 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_jaeger_query": { 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_jaeger_query", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Jaeger query container image", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.626 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_keepalived": { 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_keepalived", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Keepalived container image", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.627 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_loki": { 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_loki", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.628 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Loki container image", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_node_exporter": { 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_node_exporter", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.629 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_nvmeof": { 2024-04-06T14:21:08.630 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_nvmeof", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/ceph/nvmeof:1.0.0", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Nvme-of container image", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_prometheus": { 2024-04-06T14:21:08.631 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_prometheus", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Prometheus container image", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.632 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_promtail": { 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_promtail", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Promtail container image", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "container_image_snmp_gateway": { 2024-04-06T14:21:08.633 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_image_snmp_gateway", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "SNMP Gateway container image", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "container_init": { 2024-04-06T14:21:08.634 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "container_init", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Run podman/docker with `--init`", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "daemon_cache_timeout": { 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "daemon_cache_timeout", 2024-04-06T14:21:08.635 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache service (daemon) inventory", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "default_cephadm_command_timeout": { 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_cephadm_command_timeout", 2024-04-06T14:21:08.636 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "900", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "default_registry": { 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "default_registry", 2024-04-06T14:21:08.637 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "docker.io", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "device_cache_timeout": { 2024-04-06T14:21:08.638 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_cache_timeout", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1800", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache device inventory", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "device_enhanced_scan": { 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_enhanced_scan", 2024-04-06T14:21:08.639 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use libstoragemgmt during device scans", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "facts_cache_timeout": { 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "facts_cache_timeout", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.640 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "seconds to cache host facts data", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "host_check_interval": { 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "host_check_interval", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.641 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to perform a host check", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "hw_monitoring": { 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hw_monitoring", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.642 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Deploy hw monitoring daemon on every host.", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "inventory_list_all": { 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "inventory_list_all", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.643 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.644 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "log_refresh_metadata": { 2024-04-06T14:21:08.645 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_refresh_metadata", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.646 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.647 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.648 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.649 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf": { 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.650 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "*", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.651 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "max_count_per_host": { 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_count_per_host", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.652 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of daemons per service per host", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "max_osd_draining_count": { 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_osd_draining_count", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.653 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "migration_current": { 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "migration_current", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.654 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "internal - do not modify", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "mode": { 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mode", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "root", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.655 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm-package", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "root" 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "mode for remote execution of cephadm", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "oob_default_addr": { 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "oob_default_addr", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "169.254.1.1", 2024-04-06T14:21:08.656 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Default address for RedFish API (oob management).", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "prometheus_alerts_path": { 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus_alerts_path", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.657 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "location of alerts to include in prometheus deployments", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_insecure": { 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_insecure", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.658 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_password": { 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_password", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.659 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_url": { 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_url", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.660 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "registry_username": { 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "registry_username", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.661 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "secure_monitoring_stack": { 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "secure_monitoring_stack", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.662 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "service_discovery_port": { 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "service_discovery_port", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.663 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8765", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "cephadm service discovery port", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "ssh_config_file": { 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssh_config_file", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.664 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "customized SSH config file to connect to managed hosts", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "use_agent": { 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_agent", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.665 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "use_repo_digest": { 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "use_repo_digest", 2024-04-06T14:21:08.666 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_failed_host_check": { 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_failed_host_check", 2024-04-06T14:21:08.667 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if the host check fails", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_daemons": { 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_daemons", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.668 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_on_stray_hosts": { 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_on_stray_hosts", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.669 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crash", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.670 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.671 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.672 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.673 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.674 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.675 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "retain_interval": { 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retain_interval", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "31536000", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain crashes before pruning them", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_recent_interval": { 2024-04-06T14:21:08.676 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_recent_interval", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1209600", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "time interval in which to warn about recent crashes", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.677 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dashboard", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.678 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_HOST": { 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_HOST", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.679 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.680 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_ENABLED": { 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_ENABLED", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.681 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "AUDIT_API_LOG_PAYLOAD": { 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.682 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "ENABLE_BROWSABLE_API": { 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ENABLE_BROWSABLE_API", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.683 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_CEPHFS": { 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.684 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.685 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_ISCSI": { 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.686 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_MIRRORING": { 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.687 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_NFS": { 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_NFS", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.688 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RBD": { 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RBD", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.689 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "FEATURE_TOGGLE_RGW": { 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "FEATURE_TOGGLE_RGW", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.690 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.691 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_PASSWORD": { 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_PASSWORD", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.692 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_SSL_VERIFY": { 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.693 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_URL": { 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_URL", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.694 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_API_USERNAME": { 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_API_USERNAME", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.695 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_FRONTEND_API_URL": { 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.696 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.697 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "ISCSI_API_SSL_VERIFICATION": { 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.698 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "ISSUE_TRACKER_API_KEY": { 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ISSUE_TRACKER_API_KEY", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.699 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_HOST": { 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_HOST", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2024-04-06T14:21:08.700 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2024-04-06T14:21:08.701 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.702 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.703 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2024-04-06T14:21:08.704 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2024-04-06T14:21:08.705 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2024-04-06T14:21:08.706 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2024-04-06T14:21:08.707 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_ENABLED": { 2024-04-06T14:21:08.708 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_ENABLED", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2024-04-06T14:21:08.709 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.710 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.711 INFO:teuthology.orchestra.run.smithi090.stdout: "PWD_POLICY_MIN_LENGTH": { 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.712 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "REST_REQUESTS_TIMEOUT": { 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "REST_REQUESTS_TIMEOUT", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "45", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.713 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ACCESS_KEY": { 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ACCESS_KEY", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.714 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_ADMIN_RESOURCE": { 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "admin", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.715 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SECRET_KEY": { 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SECRET_KEY", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.716 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "RGW_API_SSL_VERIFY": { 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "RGW_API_SSL_VERIFY", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.717 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "UNSAFE_TLS_v1_2": { 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "UNSAFE_TLS_v1_2", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.718 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_SPAN": { 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.719 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.720 INFO:teuthology.orchestra.run.smithi090.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.721 INFO:teuthology.orchestra.run.smithi090.stdout: "cross_origin_url": { 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cross_origin_url", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.722 INFO:teuthology.orchestra.run.smithi090.stdout: "crt_file": { 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "crt_file", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.723 INFO:teuthology.orchestra.run.smithi090.stdout: "debug": { 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "debug", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Enable/disable debug options", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.724 INFO:teuthology.orchestra.run.smithi090.stdout: "jwt_token_ttl": { 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "jwt_token_ttl", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "28800", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:08.725 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.726 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.727 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.728 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.729 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.730 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "motd": { 2024-04-06T14:21:08.731 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "motd", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "The message of the day", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect_resolve_ip_addr": { 2024-04-06T14:21:08.732 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "redirect_resolve_ip_addr", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.733 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.734 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8080", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:08.735 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl_server_port": { 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl_server_port", 2024-04-06T14:21:08.736 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8443", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:08.737 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "redirect", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "redirect" 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.738 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.739 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "url_prefix": { 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url_prefix", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.740 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "devicehealth", 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_monitoring": { 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_monitoring", 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.741 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "monitor device health metrics", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.742 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.743 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.744 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.745 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.746 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "mark_out_threshold": { 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mark_out_threshold", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "2419200", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "automatically mark OSD if it may fail before this long", 2024-04-06T14:21:08.747 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "pool_name": { 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pool_name", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "device_health_metrics", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name of pool in which to store device health metrics", 2024-04-06T14:21:08.748 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "retention_period": { 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "retention_period", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15552000", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long to retain device health metrics", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.749 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_frequency": { 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_frequency", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to scrape device health metrics", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.750 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "self_heal": { 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "self_heal", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "preemptively heal cluster around devices that may fail", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.751 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how frequently to wake up and check device health", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.752 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "warn_threshold": { 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "warn_threshold", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7257600", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "raise health warning if OSD may fail before this long", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.753 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "diskprediction_local", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.754 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.755 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.756 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.757 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.758 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "predict_interval": { 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predict_interval", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "86400", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.759 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "predictor_model": { 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "predictor_model", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "prophetstor", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.760 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "600", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.761 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "influx", 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": false, 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "influxdb python module not found", 2024-04-06T14:21:08.762 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "batch_size": { 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "batch_size", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5000", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.763 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "database": { 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "database", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "ceph", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.764 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "hostname": { 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "hostname", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server hostname", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.765 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "30", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "5", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.766 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.767 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.768 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.769 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.770 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.771 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "password": { 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "password", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "password of InfluxDB server user", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.772 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "port": { 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "port", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "8086", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "InfluxDB server port", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.773 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "ssl": { 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ssl", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "false", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:08.774 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "threads": { 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threads", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "32", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2024-04-06T14:21:08.775 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "username": { 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "username", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.776 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "username of InfluxDB server user", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "verify_ssl": { 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "verify_ssl", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "true", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.777 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "insights", 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.778 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.779 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.780 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.781 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.782 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.783 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "iostat", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.784 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.785 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.786 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.787 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.788 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "k8sevents", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.789 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "ceph_event_retention_days": { 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "ceph_event_retention_days", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "7", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Days to hold ceph event information within local cache", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.790 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "config_check_secs": { 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "config_check_secs", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "10", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.791 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.792 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.793 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.794 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.795 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.796 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "localpool", 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "failure_domain": { 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "failure_domain", 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.797 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "host", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "failure domain for any created local pool", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.798 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.799 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.800 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.801 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.802 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "min_size": { 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "min_size", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.803 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default min_size for any created local pool", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "num_rep": { 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "num_rep", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3", 2024-04-06T14:21:08.804 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default replica count for any created local pool", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_num": { 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_num", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "128", 2024-04-06T14:21:08.805 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "default pg_num for any created local pool", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "prefix": { 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prefix", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.806 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "name prefix for any created local pool", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "subtree": { 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "subtree", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "rack", 2024-04-06T14:21:08.807 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "CRUSH level for which to create a local pool", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mds_autoscaler", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.808 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.809 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.810 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.811 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.812 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.813 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirroring", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.814 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.815 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.816 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.817 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.818 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "nfs", 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.819 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.820 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.821 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.822 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.823 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.824 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "fail_fs": { 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "fail_fs", 2024-04-06T14:21:08.825 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.826 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.827 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.828 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.829 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.830 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.831 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator": { 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "orchestrator", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "cephadm", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "rook", 2024-04-06T14:21:08.832 INFO:teuthology.orchestra.run.smithi090.stdout: "test_orchestrator" 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Orchestrator backend", 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_perf_query", 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.833 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.834 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.835 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.836 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.837 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.838 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "osd_support", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.839 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.840 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.841 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.842 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.843 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "pg_autoscaler", 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.844 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.845 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.846 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.847 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.848 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.849 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.850 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "threshold": { 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "threshold", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "3.0", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1.0", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "scaling threshold", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.851 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "progress", 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_pg_recovery_event": { 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_pg_recovery_event", 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.852 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow the module to show pg recovery progress", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.853 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.854 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.855 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.856 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.857 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.858 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "max_completed_events": { 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_completed_events", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "50", 2024-04-06T14:21:08.859 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "number of past completed events to remember", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "sleep_interval": { 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "sleep_interval", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "5", 2024-04-06T14:21:08.860 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "how long the module is going to sleep", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "prometheus", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.861 INFO:teuthology.orchestra.run.smithi090.stdout: "cache": { 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "cache", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.862 INFO:teuthology.orchestra.run.smithi090.stdout: "exclude_perf_counters": { 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "exclude_perf_counters", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Do not include perf-counters in the metrics output", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.863 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.864 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.865 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.866 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.867 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.868 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools": { 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.869 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_stats_pools_refresh_interval": { 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_stats_pools_refresh_interval", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "300", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.870 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "scrape_interval": { 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "scrape_interval", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15.0", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.871 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "::", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.872 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "9283", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "the port on which the module listens for HTTP requests", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.873 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "stale_cache_strategy": { 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stale_cache_strategy", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "log", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.874 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_behaviour": { 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_behaviour", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "default", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "default", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "error" 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.875 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "standby_error_status_code": { 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "standby_error_status_code", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "500", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "400", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "599", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.876 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_support", 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.877 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.878 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.879 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.880 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.881 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_snap_create": { 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_snap_create", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.882 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "mirror_snapshot_schedule": { 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "mirror_snapshot_schedule", 2024-04-06T14:21:08.883 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "trash_purge_schedule": { 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "trash_purge_schedule", 2024-04-06T14:21:08.884 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.885 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "restful", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "enable_auth": { 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enable_auth", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.886 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "key_file": { 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "key_file", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.887 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.888 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.889 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.890 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.891 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.892 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "server_addr": { 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_addr", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.893 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "server_port": { 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "server_port", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.894 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rgw", 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.895 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.896 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.897 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.898 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.899 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.900 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rook", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.901 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.902 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.903 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.904 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.905 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "storage_class": { 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "storage_class", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "local", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.906 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "storage class name for LSO-discovered PVs", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "selftest", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.907 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.908 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.909 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.910 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.911 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "roption1": { 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption1", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.912 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "roption2": { 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "roption2", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.913 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "xyz", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption1": { 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption1", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.914 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption2": { 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption2", 2024-04-06T14:21:08.915 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption3": { 2024-04-06T14:21:08.916 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption3", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "float", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.917 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption4": { 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption4", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.918 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption5": { 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption5", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.919 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption6": { 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption6", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.920 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "rwoption7": { 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rwoption7", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "1", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "42", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.921 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "testkey": { 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testkey", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.922 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "testlkey": { 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testlkey", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.923 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "testnewline": { 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "testnewline", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.924 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snap_schedule", 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "allow_m_granularity": { 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "allow_m_granularity", 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.925 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "allow minute scheduled snapshots", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "dump_on_update": { 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "dump_on_update", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.926 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "dump database to debug log on update", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.927 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.928 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.929 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.930 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.931 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "stats", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.932 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.933 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.934 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.935 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.936 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.937 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "status", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.938 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.939 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.940 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.941 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.942 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telegraf", 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.943 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "address": { 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "address", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.944 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "15", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.945 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.946 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.947 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.948 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.949 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.950 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "telemetry", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_basic": { 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_basic", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.951 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share basic cluster information (size, version)", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_crash": { 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_crash", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.952 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_device": { 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_device", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.953 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_ident": { 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_ident", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.954 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "channel_perf": { 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "channel_perf", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.955 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Share various performance metrics of a cluster", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "contact": { 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "contact", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.956 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "description": { 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "description", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.957 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "device_url": { 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "device_url", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/device", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.958 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "enabled": { 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "enabled", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.959 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "24", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "8", 2024-04-06T14:21:08.960 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "last_opt_revision": { 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "last_opt_revision", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "1", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.961 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard": { 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.962 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "leaderboard_description": { 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "leaderboard_description", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.963 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.964 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.965 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.966 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.967 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.968 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "organization": { 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "organization", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.969 INFO:teuthology.orchestra.run.smithi090.stdout: "proxy": { 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "proxy", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.970 INFO:teuthology.orchestra.run.smithi090.stdout: "url": { 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "url", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "https://telemetry.ceph.com/report", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.971 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "test_orchestrator", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.972 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.973 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.974 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.975 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.976 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "volumes", 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.977 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.978 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.979 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.980 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.981 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "max_concurrent_clones": { 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "max_concurrent_clones", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.982 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "4", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Number of asynchronous cloner threads", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_delay": { 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_delay", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.983 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "0", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "snapshot_clone_no_wait": { 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snapshot_clone_no_wait", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.984 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "True", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "Reject subvolume clone request when cloner threads are busy", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix", 2024-04-06T14:21:08.985 INFO:teuthology.orchestra.run.smithi090.stdout: "can_run": true, 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "error_string": "", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "module_options": { 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "discovery_interval": { 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "discovery_interval", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "uint", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "100", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.986 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "identifier": { 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "identifier", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.987 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "interval": { 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "interval", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "secs", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "60", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.988 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "log_level": { 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_level", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.989 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster": { 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.990 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_cluster_level": { 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_cluster_level", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "info", 2024-04-06T14:21:08.991 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [ 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "critical", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "debug", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "error", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "info", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "warning" 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.992 INFO:teuthology.orchestra.run.smithi090.stdout: "log_to_file": { 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "log_to_file", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "bool", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 1, 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "False", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.993 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_host": { 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_host", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.994 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_port": { 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_port", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "int", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "10051", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.995 INFO:teuthology.orchestra.run.smithi090.stdout: "zabbix_sender": { 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "zabbix_sender", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "str", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "level": "advanced", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "flags": 0, 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "default_value": "/usr/bin/zabbix_sender", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "min": "", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "max": "", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "enum_allowed": [], 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "desc": "", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "long_desc": "", 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "tags": [], 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: "see_also": [] 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.996 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "services": { 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "dashboard": "https://172.21.15.90:8443/" 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "always_on_modules": { 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "octopus": [ 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "balancer", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "crash", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "devicehealth", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_autoscaler", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "progress", 2024-04-06T14:21:08.997 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_support", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "status", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "telemetry", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "volumes" 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "pacific": [ 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "balancer", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "crash", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "devicehealth", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_autoscaler", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "progress", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_support", 2024-04-06T14:21:08.998 INFO:teuthology.orchestra.run.smithi090.stdout: "status", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "telemetry", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "volumes" 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "quincy": [ 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "balancer", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "crash", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "devicehealth", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_autoscaler", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "progress", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_support", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "status", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "telemetry", 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "volumes" 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: ], 2024-04-06T14:21:08.999 INFO:teuthology.orchestra.run.smithi090.stdout: "reef": [ 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "balancer", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "crash", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "devicehealth", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "orchestrator", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "pg_autoscaler", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "progress", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "rbd_support", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "status", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "telemetry", 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "volumes" 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "last_failure_osd_epoch": 84, 2024-04-06T14:21:09.000 INFO:teuthology.orchestra.run.smithi090.stdout: "active_clients": [ 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "devicehealth", 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 3650151185 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "libcephsqlite", 2024-04-06T14:21:09.001 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 2884357184 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "rbd_support", 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.002 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 4077277138 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snap_schedule", 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 144435379 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.003 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "snap_schedule", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 2183800872 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "name": "volumes", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "addrvec": [ 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: { 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "type": "v2", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "addr": "172.21.15.90:0", 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: "nonce": 1341986101 2024-04-06T14:21:09.004 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.005 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.005 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:09.005 INFO:teuthology.orchestra.run.smithi090.stdout: ] 2024-04-06T14:21:09.005 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:09.006 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:08 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3459786841' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:09.006 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:08 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2276685300' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-06T14:21:09.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:08 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3459786841' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:09.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:08 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2276685300' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-06T14:21:09.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 msgr2=0x7f27740a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:09.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 0x7f27740a3750 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f2784005950 tx=0x7f27840058e0 comp rx=0 tx=0).stop 2024-04-06T14:21:09.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 msgr2=0x7f2788197030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:09.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 secure :-1 s=READY pgs=921 cs=0 l=1 rev1=1 crypto rx=0x7f2778009fd0 tx=0x7f277800f670 comp rx=0 tx=0).stop 2024-04-06T14:21:09.007 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 shutdown_connections 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f27740a1290 0x7f27740a3750 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f278810b210 0x7f278819b910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2788101df0 0x7f2788197030 unknown :-1 s=CLOSED pgs=921 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 --2- 172.21.15.90:0/2276685300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2788102710 0x7f2788197570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.713+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 >> 172.21.15.90:0/2276685300 conn(0x7f27880fb650 msgr2=0x7f27880fcf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.714+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 shutdown_connections 2024-04-06T14:21:09.008 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:07.714+0000 7f278fdf1700 1 -- 172.21.15.90:0/2276685300 wait complete. 2024-04-06T14:21:09.013 DEBUG:tasks.check_counter:Getting stats from a 2024-04-06T14:21:09.014 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.a perf dump 2024-04-06T14:21:09.014 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:08 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3459786841' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:09.014 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:08 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2276685300' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2024-04-06T14:21:09.276 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:21:09.446 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:09 smithi090 ceph-mon[43420]: pgmap v494: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 496 KiB/s rd, 6.7 MiB/s wr, 632 op/s 2024-04-06T14:21:09.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:09 smithi105 ceph-mon[46969]: pgmap v494: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 496 KiB/s rd, 6.7 MiB/s wr, 632 op/s 2024-04-06T14:21:09.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:09 smithi134 ceph-mon[47549]: pgmap v494: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.5 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 496 KiB/s rd, 6.7 MiB/s wr, 632 op/s 2024-04-06T14:21:10.508 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 0, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 2, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 0, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 0, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.001365916, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.000000000, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.000321245, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:10.509 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.510 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 287, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 277, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 1442176, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 158432, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.043205151, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.013691652, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.030558150, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.006186404, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 276, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.064071237, 2024-04-06T14:21:10.511 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000232142 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 1442176, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 158432 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 7, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 215, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 3520, 2024-04-06T14:21:10.512 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 266656, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.014005183, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.009141204, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.002656326, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 213, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.122156251, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000573503 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.513 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 3520, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 266656 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "cct": { 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "total_workers": 0, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "unhealthy_workers": 0 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "mempool": { 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:10.514 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:10.515 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:10.516 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_bytes": 147947, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_items": 29, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_items": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:10.517 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_items": 2, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_items": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:10.518 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_items": 0 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "val": 0, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "max": 104857600, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "get_started": 0, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "get": 290, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "get_sum": 1401660, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_success": 290, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "take": 0, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "take_sum": 0, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "put": 290, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "put_sum": 1401660, 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "wait": { 2024-04-06T14:21:10.519 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:10.520 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:10.520 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:10.520 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:10.520 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:10.520 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.a 2024-04-06T14:21:10.882 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.a 2024-04-06T14:21:10.882 DEBUG:tasks.check_counter:Getting stats from d 2024-04-06T14:21:10.882 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.d perf dump 2024-04-06T14:21:11.107 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:21:11.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:11 smithi105 ceph-mon[46969]: pgmap v495: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 2.6 MiB/s wr, 271 op/s 2024-04-06T14:21:11.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:11 smithi090 ceph-mon[43420]: pgmap v495: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 2.6 MiB/s wr, 271 op/s 2024-04-06T14:21:11.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:11 smithi134 ceph-mon[47549]: pgmap v495: 97 pgs: 2 active+clean+snaptrim, 6 active+clean+snaptrim_wait, 89 active+clean; 5.3 GiB data, 22 GiB used, 1.0 TiB / 1.0 TiB avail; 96 KiB/s rd, 2.6 MiB/s wr, 271 op/s 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 4, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 4, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 3040, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 448, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.001904446, 2024-04-06T14:21:12.159 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.000042212, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.000922037, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 3, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000969007, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000323002 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:12.160 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 3040, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 448 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 3, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 215, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 480, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 266560, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.013615694, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.009044369, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.002329599, 2024-04-06T14:21:12.161 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 212, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.085876659, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000405078 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 480, 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 266560 2024-04-06T14:21:12.162 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 290, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 280, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 1442992, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 160208, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.044232471, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.013881824, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.031501841, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.006230691, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 279, 2024-04-06T14:21:12.163 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.057527285, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000206190 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 1442992, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 160208 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "cct": { 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "total_workers": 0, 2024-04-06T14:21:12.164 INFO:teuthology.orchestra.run.smithi090.stdout: "unhealthy_workers": 0 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "mempool": { 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:12.165 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:12.166 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:12.167 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_bytes": 147970, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_items": 29, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:12.168 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_items": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_items": 2, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_items": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_items": 0 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "val": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "max": 104857600, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "get_started": 0, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "get": 293, 2024-04-06T14:21:12.169 INFO:teuthology.orchestra.run.smithi090.stdout: "get_sum": 1402050, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_success": 293, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "take": 0, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "take_sum": 0, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "put": 293, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "put_sum": 1402050, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "wait": { 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:12.170 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:12.549 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.d 2024-04-06T14:21:12.549 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.d 2024-04-06T14:21:12.550 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.d 2024-04-06T14:21:12.550 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.d 2024-04-06T14:21:12.550 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.d 2024-04-06T14:21:12.550 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.d 2024-04-06T14:21:12.550 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.d 2024-04-06T14:21:12.550 DEBUG:tasks.check_counter:Getting stats from g 2024-04-06T14:21:12.550 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.g perf dump 2024-04-06T14:21:12.780 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:21:13.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:13 smithi090 ceph-mon[43420]: pgmap v496: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 83 KiB/s rd, 106 KiB/s wr, 12 op/s 2024-04-06T14:21:13.248 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:13 smithi090 ceph-mon[43420]: osdmap e104: 12 total, 12 up, 12 in 2024-04-06T14:21:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:13 smithi134 ceph-mon[47549]: pgmap v496: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 83 KiB/s rd, 106 KiB/s wr, 12 op/s 2024-04-06T14:21:13.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:13 smithi134 ceph-mon[47549]: osdmap e104: 12 total, 12 up, 12 in 2024-04-06T14:21:13.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:13 smithi105 ceph-mon[46969]: pgmap v496: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 83 KiB/s rd, 106 KiB/s wr, 12 op/s 2024-04-06T14:21:13.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:13 smithi105 ceph-mon[46969]: osdmap e104: 12 total, 12 up, 12 in 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 0, 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 2, 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 0, 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 0, 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:13.869 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.001436875, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.000000000, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.000385587, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:13.870 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 7, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 215, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 3520, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 266656, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:13.871 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.014598947, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.009282130, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.002969626, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 213, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.045226631, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000212331 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:13.872 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 3520, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 266656 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 288, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 278, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 1442448, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 159024, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.043059289, 2024-04-06T14:21:13.873 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.014249560, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.028797580, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.006221093, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 277, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.072201129, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000260653 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:13.874 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 1442448, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 159024 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "cct": { 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "total_workers": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "unhealthy_workers": 0 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "mempool": { 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:13.875 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:13.876 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:13.877 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_items": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_bytes": 147977, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_items": 29, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:13.878 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_items": 0, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:13.879 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_items": 2, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_items": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_items": 0 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "val": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "max": 104857600, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "get_started": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "get": 291, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "get_sum": 1401790, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_success": 291, 2024-04-06T14:21:13.880 INFO:teuthology.orchestra.run.smithi090.stdout: "take": 0, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "take_sum": 0, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "put": 291, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "put_sum": 1401790, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "wait": { 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:13.881 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:14.240 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.g 2024-04-06T14:21:14.240 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.g 2024-04-06T14:21:14.240 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.g 2024-04-06T14:21:14.240 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.g 2024-04-06T14:21:14.240 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.g 2024-04-06T14:21:14.241 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.g 2024-04-06T14:21:14.241 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.g 2024-04-06T14:21:14.241 DEBUG:tasks.check_counter:Getting stats from j 2024-04-06T14:21:14.241 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.j perf dump 2024-04-06T14:21:14.464 INFO:teuthology.orchestra.run.smithi090.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-06T14:21:15.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:15 smithi090 ceph-mon[43420]: pgmap v498: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 59 KiB/s wr, 10 op/s 2024-04-06T14:21:15.560 INFO:teuthology.orchestra.run.smithi090.stdout:{ 2024-04-06T14:21:15.560 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:15.560 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 0, 2024-04-06T14:21:15.560 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 2, 2024-04-06T14:21:15.560 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 0, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 0, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.001352452, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.000000000, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.000317031, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:15.561 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 295, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 283, 2024-04-06T14:21:15.562 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 1446304, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 161248, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.045515196, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.014397957, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.032309758, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.006567342, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 283, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.047488142, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000167802 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:15.563 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 1446304, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 161248 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_messages": 3, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages": 217, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_bytes": 480, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_bytes": 266912, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_total_time": 0.013773087, 2024-04-06T14:21:15.564 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_send_time": 0.009516240, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_recv_time": 0.002043511, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 214, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.078372188, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000366225 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_recv_encrypted_bytes": 480, 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: "msgr_send_encrypted_bytes": 266912 2024-04-06T14:21:15.565 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "cct": { 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "total_workers": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "unhealthy_workers": 0 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "mempool": { 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:15.566 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:15.567 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:15.568 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_bytes": 149001, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_anon_items": 30, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_items": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:15.569 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_items": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "pgmap_items": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "mds_co_items": 2, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_1_items": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "unittest_2_items": 0 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: }, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "val": 0, 2024-04-06T14:21:15.570 INFO:teuthology.orchestra.run.smithi090.stdout: "max": 104857600, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "get_started": 0, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "get": 294, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "get_sum": 1402180, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "get_or_fail_success": 294, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "take": 0, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "take_sum": 0, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "put": 294, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "put_sum": 1402180, 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "wait": { 2024-04-06T14:21:15.571 INFO:teuthology.orchestra.run.smithi090.stdout: "avgcount": 0, 2024-04-06T14:21:15.572 INFO:teuthology.orchestra.run.smithi090.stdout: "sum": 0.000000000, 2024-04-06T14:21:15.572 INFO:teuthology.orchestra.run.smithi090.stdout: "avgtime": 0.000000000 2024-04-06T14:21:15.572 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:15.572 INFO:teuthology.orchestra.run.smithi090.stdout: } 2024-04-06T14:21:15.572 INFO:teuthology.orchestra.run.smithi090.stdout:} 2024-04-06T14:21:15.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:15 smithi134 ceph-mon[47549]: pgmap v498: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 59 KiB/s wr, 10 op/s 2024-04-06T14:21:15.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:15 smithi105 ceph-mon[46969]: pgmap v498: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 59 KiB/s wr, 10 op/s 2024-04-06T14:21:15.923 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.j 2024-04-06T14:21:15.924 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.j 2024-04-06T14:21:15.924 DEBUG:tasks.check_counter:Getting stats from b 2024-04-06T14:21:15.924 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.b perf dump 2024-04-06T14:21:16.363 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:21:17.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:17 smithi105 ceph-mon[46969]: pgmap v499: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 48 KiB/s wr, 8 op/s 2024-04-06T14:21:17.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:17 smithi090 ceph-mon[43420]: pgmap v499: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 48 KiB/s wr, 8 op/s 2024-04-06T14:21:17.642 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:17 smithi134 ceph-mon[47549]: pgmap v499: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 48 KiB/s wr, 8 op/s 2024-04-06T14:21:17.684 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 13277, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 13368, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 5199114, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 45970963, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 15, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 4, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.897790479, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.304760125, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.276234995, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.276323795, 2024-04-06T14:21:17.685 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 13367, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 1.793836036, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000134198 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 5281, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000976529, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000184 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 2035504, 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 362896 2024-04-06T14:21:17.686 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 16480, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 16548, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 4219132, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 33726847, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 234, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 7, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.848507698, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.241772682, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.231066582, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.311659469, 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:17.687 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 16544, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 1.970041290, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000119078 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 496, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000069701, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000140 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 91872, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 656000 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 25380, 2024-04-06T14:21:17.688 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 29256, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 8012650, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 31907769, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 13, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 6, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 1.538463445, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.587359288, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 1.301768252, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.279547168, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 29255, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 2.649834075, 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000090577 2024-04-06T14:21:17.689 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 33681, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.005257691, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000156 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 5269984, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 11345920 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "cct": { 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "total_workers": 1, 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: "unhealthy_workers": 0 2024-04-06T14:21:17.690 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-MDSRank": { 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 28475, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 8.539583479, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000299897 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 1014, 2024-04-06T14:21:17.691 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.038616714, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000038083 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "request": 7147, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "reply": 7147, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "reply_latency": { 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 7147, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 17.256766706, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.002414546 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.692 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_reply": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "forward": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_commit": 48, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_split": 4, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_merge": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes": 2922, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_top": 34, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_bottom": 2888, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:17.693 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pinned": 2873, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_expired": 0, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_with_caps": 1074, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "caps": 1074, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "subtrees": 5, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse": 15232, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_hit": 12544, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_forward": 0, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_discover": 0, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:17.694 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_lock": 852, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "load_cent": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "q": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "exported": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "exported_inodes": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "imported": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "imported_inodes": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_backtrace_fetch": 29037, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rfiles": 1038, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rbytes": 1742189200, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rsnaps": 6, 2024-04-06T14:21:17.695 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_fetch": 29037, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_inodes": 979, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_base_inodes": 28, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dirfrag_rstats": 951, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_file_inodes": 28058, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_revoke": 1660, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_grant": 4066, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flushsnap_ack": 85, 2024-04-06T14:21:17.696 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps": 6613, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps_dirty": 3257, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_cap_release": 181, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "process_request_cap_release": 2515 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_cache": { 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update": 81, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update_receipt": 0, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_try_discover": 0, 2024-04-06T14:21:17.697 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_send_discover": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_handle_discover": 121, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays": 9, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_created": 872, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_enqueued": 275, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_reintegrated": 588, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_migrated": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:17.698 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_started": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_completed": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_enqueue_scrub": 28, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_flush": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragmentdir": 4, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_inodestats": 0 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_log": { 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "evadd": 9680, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "evex": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "evtrm": 0, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "ev": 9680, 2024-04-06T14:21:17.699 INFO:teuthology.orchestra.run.smithi105.stdout: "evexg": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "evexd": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "segadd": 24, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "segex": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "segtrm": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "seg": 24, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "segexg": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "segexd": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "expos": 4194304, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "wrpos": 90336962, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "rdpos": 0, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "jlat": { 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3947, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 33.076348523, 2024-04-06T14:21:17.700 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.008380123 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "replayed": 0 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_mem": { 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "ino": 1520, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "ino+": 1823, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "ino-": 303, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dir": 75, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dir+": 79, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dir-": 4, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dn": 2922, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dn+": 3605, 2024-04-06T14:21:17.701 INFO:teuthology.orchestra.run.smithi105.stdout: "dn-": 683, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "cap": 1074, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "cap+": 1591, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "cap-": 517, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "rss": 71608, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "heap": 207156 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_server": { 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_client_request": 10138, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:17.702 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_request": 7147, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_session": 1230, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_peer_request": 0, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "req_create_latency": { 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 1325, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 3.379854090, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.002550833 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getattr_latency": { 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 197, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 2.821612207, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.014322904 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.703 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "req_link_latency": { 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 588, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 1.451495377, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.002468529 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookup_latency": { 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 2659, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 2.459971641, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000925149 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:17.704 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupino_latency": { 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupname_latency": { 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:17.705 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lssnap_latency": { 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 16, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.002348854, 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000146803 2024-04-06T14:21:17.706 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mkdir_latency": { 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 35, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.323708959, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.009248827 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mknod_latency": { 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mksnap_latency": { 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 16, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.140273343, 2024-04-06T14:21:17.707 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.008767083 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "req_open_latency": { 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 574, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.156064581, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000271889 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "req_readdir_latency": { 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 244, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.373468484, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.001530608 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rename_latency": { 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 834, 2024-04-06T14:21:17.708 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 4.764692687, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.005713060 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmdir_latency": { 2024-04-06T14:21:17.709 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 10, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.170720741, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.017072074 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.710 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setattr_latency": { 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 1, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000789445, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000789445 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.711 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setlayout_latency": { 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setxattr_latency": { 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 8, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.021172233, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.002646529 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "req_symlink_latency": { 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.712 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "req_unlink_latency": { 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 640, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 1.190594064, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.001860303 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.713 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_sessions": { 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "session_count": 2, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "session_add": 6, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "session_remove": 4, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_open": 2, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_stale": 0, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "total_load": 556, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "average_load": 278, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "avg_session_uptime": 934, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.714 INFO:teuthology.orchestra.run.smithi105.stdout: "mempool": { 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:17.715 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:17.716 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_items": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_bytes": 887442, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_items": 1419, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_bytes": 880, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_items": 10, 2024-04-06T14:21:17.717 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_bytes": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_items": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_items": 188, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_items": 0, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_bytes": 7182338, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_items": 203321, 2024-04-06T14:21:17.718 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_items": 0, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_items": 0 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "objecter": { 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_active": 0, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_laggy": 0, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send": 34535, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send_bytes": 86967153, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_resend": 37, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_reply": 34535, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "op_latency": { 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 34535, 2024-04-06T14:21:17.719 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 85.163305147, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.002465999 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op_inflight": 0, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "oplen_avg": { 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 34535, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 34880 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op": 34535, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op_r": 29222, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op_w": 5313, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op_rmw": 0, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "op_pg": 0, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_stat": 0, 2024-04-06T14:21:17.720 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_create": 11, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_read": 157, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_write": 4076, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writefull": 165, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writesame": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_append": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_zero": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_truncate": 4, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_delete": 822, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_mapext": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_getxattr": 29065, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_setxattr": 22, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:17.721 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_call": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_watch": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_notify": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_other": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_active": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_send": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_resend": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_ping": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_active": 0, 2024-04-06T14:21:17.722 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_send": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_resend": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_active": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_send": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_resend": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_active": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_send": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_resend": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "command_active": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "command_send": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "command_resend": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "map_epoch": 104, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "map_full": 0, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "map_inc": 24, 2024-04-06T14:21:17.723 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_sessions": 12, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_open": 12, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_close": 0, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_laggy": 0, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_wr": 389, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_rd": 0, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_del": 169 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "oft": { 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_objs": 1, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_updates": 0, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_removes": 0 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "osd-slow-ops": { 2024-04-06T14:21:17.724 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_ops_count": 7288 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "purge_queue": { 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops_high_water": 22, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing": 0, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_high_water": 10, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executed": 275, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:17.725 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 55133, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 11685021, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 55133, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 55133, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 11685021, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.726 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 34535, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 86995664, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 34535, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 86995664, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.727 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 1024, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 34535, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 34535, 2024-04-06T14:21:17.728 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 34535, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 34535, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 275, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 28511, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:17.729 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 275, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 129, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 28511, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle-0x558a5a01c0a0": { 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 50490, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:17.730 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 9680, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 86142658, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 9680, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 3947, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 86092168, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.731 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:17.732 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds.exported=0 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds.imported=0 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds_cache.dir_update=81 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds_cache.dir_update_receipt=0 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds.root_rsnaps=6 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds_server.req_mksnap_latency.avgcount=16 2024-04-06T14:21:18.039 INFO:tasks.check_counter:Daemon mds.b mds_server.req_rmsnap_latency.avgcount=10 2024-04-06T14:21:18.040 DEBUG:tasks.check_counter:Getting stats from e 2024-04-06T14:21:18.040 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.e perf dump 2024-04-06T14:21:18.263 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 3006, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 3006, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 853014, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 934077, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 2, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.198268323, 2024-04-06T14:21:19.184 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.054923553, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.049884108, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.078883620, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3005, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.570549864, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000189866 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 3040, 2024-04-06T14:21:19.185 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 448 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 5, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 218, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 1036, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 615692, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 7, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 4, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.010880173, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.006771092, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.001942423, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.000068534, 2024-04-06T14:21:19.186 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 215, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.066726393, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000310355 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 480, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 615008 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 321, 2024-04-06T14:21:19.187 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 310, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 1475020, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 166007, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 2, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.047633014, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.013626705, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.035689500, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.006950877, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 309, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.074236724, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000240248 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.188 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 1474752, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 165696 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "cct": { 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "total_workers": 1, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "unhealthy_workers": 0 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-MDSRank": { 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:19.189 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 4855, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.110940760, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000022850 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.190 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "request": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "reply": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "reply_latency": { 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_reply": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "forward": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_commit": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_split": 0, 2024-04-06T14:21:19.191 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_merge": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_top": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_bottom": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pinned": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_expired": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_with_caps": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "caps": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "subtrees": 1, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_hit": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_forward": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_discover": 0, 2024-04-06T14:21:19.192 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_lock": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "load_cent": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "q": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "exported": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "exported_inodes": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "imported": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "imported_inodes": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_backtrace_fetch": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rfiles": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rbytes": 0, 2024-04-06T14:21:19.193 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rsnaps": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_fetch": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_inodes": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_base_inodes": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dirfrag_rstats": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_file_inodes": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_revoke": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_grant": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-06T14:21:19.194 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps_dirty": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_cap_release": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "process_request_cap_release": 0 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_cache": { 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update_receipt": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_try_discover": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_send_discover": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_handle_discover": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:19.195 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_created": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_enqueued": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_reintegrated": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_migrated": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_started": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_completed": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_enqueue_scrub": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_flush": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragmentdir": 0, 2024-04-06T14:21:19.196 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_inodestats": 0 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_log": { 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "evadd": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "evex": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "evtrm": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "ev": 7, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "evexg": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "evexd": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "segadd": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "segex": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "segtrm": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "seg": 1, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "segexg": 0, 2024-04-06T14:21:19.197 INFO:teuthology.orchestra.run.smithi105.stdout: "segexd": 0, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "expos": 4194304, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "wrpos": 4196619, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "rdpos": 4196478, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "jlat": { 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "replayed": 7 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_mem": { 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "ino": 3, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "ino+": 3, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "ino-": 0, 2024-04-06T14:21:19.198 INFO:teuthology.orchestra.run.smithi105.stdout: "dir": 1, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dir+": 1, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dir-": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dn": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dn+": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dn-": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "cap": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "cap+": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "cap-": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "rss": 47584, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "heap": 207156 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_server": { 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_client_request": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:19.199 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_request": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_session": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_peer_request": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "req_create_latency": { 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getattr_latency": { 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.200 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "req_link_latency": { 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookup_latency": { 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:19.201 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupino_latency": { 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupname_latency": { 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.202 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lssnap_latency": { 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.203 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mkdir_latency": { 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mknod_latency": { 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mksnap_latency": { 2024-04-06T14:21:19.204 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "req_open_latency": { 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "req_readdir_latency": { 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.205 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rename_latency": { 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.206 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmdir_latency": { 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.207 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setattr_latency": { 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.208 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setlayout_latency": { 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setxattr_latency": { 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "req_symlink_latency": { 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.209 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "req_unlink_latency": { 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.210 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_sessions": { 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "session_count": 2, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "session_add": 4, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "session_remove": 2, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_open": 4, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_stale": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "total_load": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "average_load": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "avg_session_uptime": 3792, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "mempool": { 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:19.211 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:19.212 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:19.213 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_bytes": 181354, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_items": 48, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_bytes": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:19.214 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_items": 188, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_items": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_bytes": 8320, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_items": 13, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_items": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:19.215 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_items": 0 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "objecter": { 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_active": 0, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_laggy": 0, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send": 3005, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send_bytes": 21, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_resend": 3, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_reply": 3005, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_latency": { 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3005, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 14.065723911, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.004680773 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.216 INFO:teuthology.orchestra.run.smithi105.stdout: "op_inflight": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "oplen_avg": { 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3005, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 3006 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "op": 3005, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "op_r": 3005, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "op_w": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "op_rmw": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "op_pg": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_stat": 1970, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_create": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_read": 1033, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_write": 0, 2024-04-06T14:21:19.217 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writefull": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writesame": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_append": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_zero": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_truncate": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_delete": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_mapext": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_getxattr": 1, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_setxattr": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:19.218 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_call": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_watch": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_notify": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_other": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_active": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_send": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_resend": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_ping": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_active": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_send": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_resend": 0, 2024-04-06T14:21:19.219 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_active": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_send": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_resend": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_active": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_send": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_resend": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "command_active": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "command_send": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "command_resend": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "map_epoch": 104, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "map_full": 0, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "map_inc": 23, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_sessions": 4, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_open": 5, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_close": 1, 2024-04-06T14:21:19.220 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_laggy": 0, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_wr": 0, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_rd": 2, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_del": 0 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "oft": { 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_objs": 0, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_updates": 0, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_removes": 0 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "osd-slow-ops": { 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_ops_count": 0 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.221 INFO:teuthology.orchestra.run.smithi105.stdout: "purge_queue": { 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops_high_water": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_high_water": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executed": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 3328, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 2010072, 2024-04-06T14:21:19.222 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 3328, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 3328, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 2010072, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:19.223 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 3005, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 2320, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 3005, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 2320, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.224 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 1024, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 3005, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 3005, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 3005, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 3005, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.225 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:19.226 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle-0x556ea3a70500": { 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:19.227 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:19.228 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-04-06T14:21:19.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:19 smithi105 ceph-mon[46969]: pgmap v500: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 70 KiB/s wr, 8 op/s 2024-04-06T14:21:19.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:19 smithi134 ceph-mon[47549]: pgmap v500: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 70 KiB/s wr, 8 op/s 2024-04-06T14:21:19.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:19 smithi090 ceph-mon[43420]: pgmap v500: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 70 KiB/s wr, 8 op/s 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds.exported=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds.imported=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds_cache.dir_update=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds_cache.dir_update_receipt=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds.root_rsnaps=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds_server.req_mksnap_latency.avgcount=0 2024-04-06T14:21:19.547 INFO:tasks.check_counter:Daemon mds.e mds_server.req_rmsnap_latency.avgcount=0 2024-04-06T14:21:19.547 DEBUG:tasks.check_counter:Getting stats from h 2024-04-06T14:21:19.548 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.h perf dump 2024-04-06T14:21:19.773 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 2584, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 1528, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 848835, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 652768, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 13, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 2, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.148866582, 2024-04-06T14:21:20.766 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.062276198, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.107430097, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.005215875, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 1528, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.217657608, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000142446 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3930, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000708377, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000180 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.767 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 377, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 424, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 105832, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 620315, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 59, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 5, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.049961770, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.012391142, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.039489803, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.000401457, 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:20.768 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 422, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.020032333, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000047469 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 690, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000112583, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000163 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 94528, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 164928 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 379, 2024-04-06T14:21:20.769 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 586, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 1490090, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 762127, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 18, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 6, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.062920156, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.021259420, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.041985799, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.006694559, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 583, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.146121432, 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000250637 2024-04-06T14:21:20.770 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 70, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000015775, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000225 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 1487536, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 686032 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "cct": { 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "total_workers": 1, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "unhealthy_workers": 0 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-MDSRank": { 2024-04-06T14:21:20.771 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 57, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.001618822, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000028400 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "queue_len": 0, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "complete_latency": { 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 2, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000012960, 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000006480 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.772 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "mds": { 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "request": 294, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "reply": 4, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "reply_latency": { 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 4, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.001055727, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000263931 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_reply": 0, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "forward": 290, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:20.773 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_commit": 11, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_split": 0, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_merge": 0, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes": 33, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_top": 10, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_bottom": 23, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_pinned": 18, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_expired": 0, 2024-04-06T14:21:20.774 INFO:teuthology.orchestra.run.smithi105.stdout: "inodes_with_caps": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "caps": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "subtrees": 2, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse": 356, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_hit": 48, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_forward": 287, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_discover": 19, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "traverse_lock": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "load_cent": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "q": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "exported": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "exported_inodes": 0, 2024-04-06T14:21:20.775 INFO:teuthology.orchestra.run.smithi105.stdout: "imported": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "imported_inodes": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_backtrace_fetch": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rfiles": 1038, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rbytes": 1742189200, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "root_rsnaps": 6, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_fetch": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_inodes": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dir_base_inodes": 0, 2024-04-06T14:21:20.776 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_dirfrag_rstats": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "scrub_file_inodes": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_revoke": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_grant": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_caps_dirty": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_cap_release": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "process_request_cap_release": 2 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_cache": { 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update": 0, 2024-04-06T14:21:20.777 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_update_receipt": 99, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_try_discover": 59, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_send_discover": 60, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "dir_handle_discover": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_created": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_enqueued": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_reintegrated": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "strays_migrated": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:20.778 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_started": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_completed": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_enqueue_scrub": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_flush": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragmentdir": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ireq_inodestats": 0 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_log": { 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "evadd": 7, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "evex": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "evtrm": 0, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "ev": 7, 2024-04-06T14:21:20.779 INFO:teuthology.orchestra.run.smithi105.stdout: "evexg": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "evexd": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "segadd": 1, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "segex": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "segtrm": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "seg": 1, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "segexg": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "segexd": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "expos": 4194304, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "wrpos": 4196619, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "rdpos": 0, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "jlat": { 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 7, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.023062524, 2024-04-06T14:21:20.780 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.003294646 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "replayed": 0 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_mem": { 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "ino": 36, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "ino+": 36, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "ino-": 0, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dir": 63, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dir+": 64, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dir-": 1, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dn": 33, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dn+": 33, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "dn-": 0, 2024-04-06T14:21:20.781 INFO:teuthology.orchestra.run.smithi105.stdout: "cap": 0, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "cap+": 4, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "cap-": 4, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "rss": 50152, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "heap": 198964 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_server": { 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_client_request": 294, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_request": 294, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_client_session": 32, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "handle_peer_request": 0, 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "req_create_latency": { 2024-04-06T14:21:20.782 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getattr_latency": { 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 2, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000587194, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000293597 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.783 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "req_link_latency": { 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookup_latency": { 2024-04-06T14:21:20.784 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 2, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000468533, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000234266 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupino_latency": { 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.785 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupname_latency": { 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.786 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "req_lssnap_latency": { 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mkdir_latency": { 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mknod_latency": { 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.787 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "req_mksnap_latency": { 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "req_open_latency": { 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.788 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "req_readdir_latency": { 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rename_latency": { 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.789 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmdir_latency": { 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.790 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setattr_latency": { 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.791 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.792 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setlayout_latency": { 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "req_setxattr_latency": { 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "req_symlink_latency": { 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.793 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "req_unlink_latency": { 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.794 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_sessions": { 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "session_count": 2, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "session_add": 4, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "session_remove": 2, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_open": 2, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "sessions_stale": 0, 2024-04-06T14:21:20.795 INFO:teuthology.orchestra.run.smithi105.stdout: "total_load": 3, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "average_load": 1, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "avg_session_uptime": 937, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "mempool": { 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:20.796 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:20.797 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_items": 0, 2024-04-06T14:21:20.798 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_bytes": 242734, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_items": 121, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:20.799 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_items": 188, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_items": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_bytes": 197248, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_items": 1120, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_items": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_items": 0 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "objecter": { 2024-04-06T14:21:20.800 INFO:teuthology.orchestra.run.smithi105.stdout: "op_active": 0, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_laggy": 0, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send": 46, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_send_bytes": 12932, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_resend": 27, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_reply": 46, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_latency": { 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 46, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.244995072, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.005325979 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "op_inflight": 0, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "oplen_avg": { 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 46, 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 70 2024-04-06T14:21:20.801 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "op": 46, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "op_r": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "op_w": 46, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "op_rmw": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "op_pg": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_stat": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_create": 10, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_read": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_write": 7, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writefull": 8, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_writesame": 0, 2024-04-06T14:21:20.802 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_append": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_zero": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_truncate": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_delete": 6, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_mapext": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_getxattr": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_setxattr": 20, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_call": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_watch": 0, 2024-04-06T14:21:20.803 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_notify": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "osdop_other": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_active": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_send": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_resend": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "linger_ping": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_active": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_send": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolop_resend": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_active": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_send": 0, 2024-04-06T14:21:20.804 INFO:teuthology.orchestra.run.smithi105.stdout: "poolstat_resend": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_active": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_send": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "statfs_resend": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "command_active": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "command_send": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "command_resend": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "map_epoch": 104, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "map_full": 0, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "map_inc": 23, 2024-04-06T14:21:20.805 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_sessions": 8, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_open": 12, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_session_close": 4, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_laggy": 0, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_wr": 17, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_rd": 0, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_del": 2 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "oft": { 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_objs": 1, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_updates": 0, 2024-04-06T14:21:20.806 INFO:teuthology.orchestra.run.smithi105.stdout: "omap_total_removes": 0 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "osd-slow-ops": { 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "slow_ops_count": 294 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "purge_queue": { 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_ops_high_water": 0, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing": 0, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executing_high_water": 0, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_executed": 0, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.807 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 3336, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 2109135, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 3336, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 3336, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 2109135, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.808 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 46, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 12932, 2024-04-06T14:21:20.809 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 46, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 12932, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 1024, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:20.810 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 46, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 46, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 46, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 46, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.811 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 0, 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:20.812 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-write_buf_throttle-0x56431e6580a0": { 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 3758096384, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 7, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 2315, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 7, 2024-04-06T14:21:20.813 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 7, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 2315, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:20.814 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds.exported=0 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds.imported=0 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds_cache.dir_update=0 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds_cache.dir_update_receipt=99 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds.root_rsnaps=6 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds_server.req_mksnap_latency.avgcount=0 2024-04-06T14:21:21.123 INFO:tasks.check_counter:Daemon mds.h mds_server.req_rmsnap_latency.avgcount=0 2024-04-06T14:21:21.123 DEBUG:tasks.check_counter:Getting stats from k 2024-04-06T14:21:21.123 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.k perf dump 2024-04-06T14:21:21.351 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-06T14:21:21.495 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:21 smithi105 ceph-mon[46969]: pgmap v501: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 54 KiB/s wr, 8 op/s 2024-04-06T14:21:21.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:21 smithi090 ceph-mon[43420]: pgmap v501: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 54 KiB/s wr, 8 op/s 2024-04-06T14:21:21.643 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:21 smithi134 ceph-mon[47549]: pgmap v501: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 54 KiB/s wr, 8 op/s 2024-04-06T14:21:22.316 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-04-06T14:21:22.316 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:22.316 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 4, 2024-04-06T14:21:22.316 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 4, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 3040, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 448, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.001397313, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.000041792, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.000510414, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 3, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000687487, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000229162 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.317 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 3040, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 448 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 294, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 284, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 1444080, 2024-04-06T14:21:22.318 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 162592, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.044869166, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.013586539, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.032106180, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.006602833, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 283, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.052291058, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000184774 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.319 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 1444080, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 162592 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_messages": 3, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages": 218, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_bytes": 480, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_bytes": 267088, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:22.320 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_total_time": 0.010906701, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_send_time": 0.006705332, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_recv_time": 0.002089143, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 215, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.103195530, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000479979 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_recv_encrypted_bytes": 480, 2024-04-06T14:21:22.321 INFO:teuthology.orchestra.run.smithi105.stdout: "msgr_send_encrypted_bytes": 267088 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "cct": { 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "total_workers": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "unhealthy_workers": 0 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "mempool": { 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:22.322 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:22.323 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_bytes": 147970, 2024-04-06T14:21:22.324 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_anon_items": 29, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:22.325 INFO:teuthology.orchestra.run.smithi105.stdout: "pgmap_items": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "mds_co_items": 2, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_1_items": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "unittest_2_items": 0 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "val": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "max": 104857600, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "get_started": 0, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "get": 297, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "get_sum": 1402570, 2024-04-06T14:21:22.326 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "get_or_fail_success": 297, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "take": 0, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "take_sum": 0, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "put": 297, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "put_sum": 1402570, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "wait": { 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "avgcount": 0, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "sum": 0.000000000, 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: "avgtime": 0.000000000 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-04-06T14:21:22.327 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.k 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.k 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.k 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.k 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.k 2024-04-06T14:21:22.671 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.k 2024-04-06T14:21:22.672 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.k 2024-04-06T14:21:22.672 DEBUG:tasks.check_counter:Getting stats from c 2024-04-06T14:21:22.672 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.c perf dump 2024-04-06T14:21:23.148 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:21:23.487 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:23 smithi134 ceph-mon[47549]: pgmap v502: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 61 KiB/s wr, 8 op/s 2024-04-06T14:21:23.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:23 smithi105 ceph-mon[46969]: pgmap v502: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 61 KiB/s wr, 8 op/s 2024-04-06T14:21:23.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:23 smithi090 ceph-mon[43420]: pgmap v502: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 61 KiB/s wr, 8 op/s 2024-04-06T14:21:24.211 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 2066, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 2055, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 27222054, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 708820, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 2, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.209323118, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.055567449, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.088636437, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.058058287, 2024-04-06T14:21:24.212 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 2054, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.421694650, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000205304 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 1475184, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 166432 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.213 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 857, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 1070, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 27824765, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 797011, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 8, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 6, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.087234455, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.024688418, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.033306604, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.022388676, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1067, 2024-04-06T14:21:24.214 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.229853966, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000215420 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 320, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 531040 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 854, 2024-04-06T14:21:24.215 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 854, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 33924523, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 264805, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 5, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.081907560, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.019952163, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.033960018, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.022165806, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 853, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.142287131, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000166807 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.216 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 3040, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 448 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "cct": { 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "total_workers": 1, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "unhealthy_workers": 0 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-MDSRank": { 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:24.217 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 5768, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.131058431, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000022721 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.218 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "mds": { 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "request": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "reply": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "reply_latency": { 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_reply": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "forward": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_commit": 0, 2024-04-06T14:21:24.219 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_split": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_merge": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes": 3280, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_top": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_bottom": 3280, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pinned": 3280, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_expired": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_with_caps": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "caps": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "subtrees": 2, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_hit": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_forward": 0, 2024-04-06T14:21:24.220 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_discover": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_lock": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "load_cent": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "q": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "exported": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "exported_inodes": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "imported": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "imported_inodes": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_backtrace_fetch": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:24.221 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rfiles": 1038, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rbytes": 1742189200, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rsnaps": 6, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_fetch": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_inodes": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_base_inodes": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dirfrag_rstats": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_file_inodes": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_revoke": 0, 2024-04-06T14:21:24.222 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_grant": 0, 2024-04-06T14:21:24.223 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:24.223 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-06T14:21:24.223 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:24.223 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps": 0, 2024-04-06T14:21:24.223 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps_dirty": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_cap_release": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "process_request_cap_release": 0 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_cache": { 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update_receipt": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_try_discover": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_send_discover": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_handle_discover": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:24.224 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_created": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_enqueued": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_reintegrated": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_migrated": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_started": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_completed": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_enqueue_scrub": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_flush": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragmentdir": 0, 2024-04-06T14:21:24.225 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_inodestats": 0 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_log": { 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "evadd": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "evex": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "evtrm": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "ev": 9683, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "evexg": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "evexd": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "segadd": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "segex": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "segtrm": 0, 2024-04-06T14:21:24.226 INFO:teuthology.orchestra.run.smithi134.stdout: "seg": 24, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "segexg": 0, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "segexd": 0, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "expos": 4194304, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "wrpos": 90480659, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "rdpos": 90456536, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "jlat": { 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "replayed": 9683 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_mem": { 2024-04-06T14:21:24.227 INFO:teuthology.orchestra.run.smithi134.stdout: "ino": 1520, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "ino+": 1795, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "ino-": 275, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dir": 75, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dir+": 79, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dir-": 4, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dn": 3280, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dn+": 3280, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "dn-": 0, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "cap": 0, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "cap+": 0, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "cap-": 0, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "rss": 58268, 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: "heap": 207156 2024-04-06T14:21:24.228 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_server": { 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_client_request": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_request": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_session": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_peer_request": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "req_create_latency": { 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getattr_latency": { 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.229 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "req_link_latency": { 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.230 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookup_latency": { 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupino_latency": { 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.231 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupname_latency": { 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.232 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lssnap_latency": { 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mkdir_latency": { 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mknod_latency": { 2024-04-06T14:21:24.233 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mksnap_latency": { 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "req_open_latency": { 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.234 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "req_readdir_latency": { 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rename_latency": { 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.235 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmdir_latency": { 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.236 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setattr_latency": { 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.237 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setlayout_latency": { 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.238 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setxattr_latency": { 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "req_symlink_latency": { 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "req_unlink_latency": { 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.239 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_sessions": { 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "session_count": 2, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "session_add": 6, 2024-04-06T14:21:24.240 INFO:teuthology.orchestra.run.smithi134.stdout: "session_remove": 4, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_open": 5, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_stale": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "total_load": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "average_load": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "avg_session_uptime": 7860, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "mempool": { 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:24.241 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:24.242 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:24.243 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_bytes": 316972, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_items": 2584, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:24.244 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_items": 188, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_items": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_bytes": 6942171, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_items": 190528, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_items": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_items": 0 2024-04-06T14:21:24.245 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "objecter": { 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_active": 0, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_laggy": 0, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send": 3449, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send_bytes": 26, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_resend": 4, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_reply": 3449, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_latency": { 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 3449, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 14.596666101, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.004232144 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.246 INFO:teuthology.orchestra.run.smithi134.stdout: "op_inflight": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "oplen_avg": { 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 3449, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 3450 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "op": 3449, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "op_r": 3449, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "op_w": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "op_rmw": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "op_pg": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_stat": 2027, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_create": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_read": 1419, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_write": 0, 2024-04-06T14:21:24.247 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writefull": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writesame": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_append": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_zero": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_truncate": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_delete": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_mapext": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_getxattr": 2, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_setxattr": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:24.248 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_call": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_watch": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_notify": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_other": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_active": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_send": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_resend": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_ping": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_active": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_send": 0, 2024-04-06T14:21:24.249 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_resend": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_active": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_send": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_resend": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_active": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_send": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_resend": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "command_active": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "command_send": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "command_resend": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "map_epoch": 104, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "map_full": 0, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "map_inc": 23, 2024-04-06T14:21:24.250 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_sessions": 9, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_open": 9, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_close": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_laggy": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_wr": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_rd": 2, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_del": 0 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "oft": { 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_objs": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_updates": 0, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_removes": 0 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "osd-slow-ops": { 2024-04-06T14:21:24.251 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_ops_count": 0 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "purge_queue": { 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops_high_water": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_high_water": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executed": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:24.252 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 3773, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 88612239, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 3773, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3773, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 88612239, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:24.253 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 3449, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 86531624, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3449, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 86531624, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.254 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 1024, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 3449, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 3449, 2024-04-06T14:21:24.255 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3449, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 3449, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:24.256 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle-0x563c907785a0": { 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:24.257 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:24.258 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.259 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:24.259 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds.exported=0 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds.imported=0 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds_cache.dir_update=0 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds_cache.dir_update_receipt=0 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds.root_rsnaps=6 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds_server.req_mksnap_latency.avgcount=0 2024-04-06T14:21:24.580 INFO:tasks.check_counter:Daemon mds.c mds_server.req_rmsnap_latency.avgcount=0 2024-04-06T14:21:24.580 DEBUG:tasks.check_counter:Getting stats from f 2024-04-06T14:21:24.581 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.f perf dump 2024-04-06T14:21:24.843 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:21:25.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:25 smithi105 ceph-mon[46969]: pgmap v503: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 73 KiB/s wr, 10 op/s 2024-04-06T14:21:25.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:25 smithi090 ceph-mon[43420]: pgmap v503: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 73 KiB/s wr, 10 op/s 2024-04-06T14:21:25.627 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:25 smithi134 ceph-mon[47549]: pgmap v503: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 48 KiB/s rd, 73 KiB/s wr, 10 op/s 2024-04-06T14:21:25.824 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 1033, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 1035, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 273488, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 321327, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 7, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 2, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.075292035, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.021393311, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.020848024, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.024792181, 2024-04-06T14:21:25.825 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1033, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.155131541, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000150175 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:25.826 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 1273, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 1261, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 1709831, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 462335, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.122701780, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.031573835, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.053387214, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.040273699, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1260, 2024-04-06T14:21:25.827 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.258616107, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000205250 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 1475296, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 166880 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 1029, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 1242, 2024-04-06T14:21:25.828 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 347088, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 851561, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 6, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 4, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.109443411, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.035477250, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.027952858, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.034749919, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1240, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.353636235, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000285190 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.829 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 3360, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 533408 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "cct": { 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "total_workers": 1, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "unhealthy_workers": 0 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-MDSRank": { 2024-04-06T14:21:25.830 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 4951, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.125579470, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000025364 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.831 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "mds": { 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "request": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "reply": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "reply_latency": { 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_reply": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "forward": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:25.832 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_commit": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_split": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_merge": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_top": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_bottom": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pinned": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_expired": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_with_caps": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "caps": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "subtrees": 1, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_hit": 0, 2024-04-06T14:21:25.833 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_forward": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_discover": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_lock": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "load_cent": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "q": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "exported": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "exported_inodes": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "imported": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "imported_inodes": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:25.834 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_backtrace_fetch": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rfiles": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rbytes": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rsnaps": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_fetch": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_inodes": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_base_inodes": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dirfrag_rstats": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_file_inodes": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:25.835 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_revoke": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_grant": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps_dirty": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_cap_release": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "process_request_cap_release": 0 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_cache": { 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update_receipt": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_try_discover": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_send_discover": 0, 2024-04-06T14:21:25.836 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_handle_discover": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_created": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_enqueued": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_reintegrated": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_migrated": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_started": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_completed": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_enqueue_scrub": 0, 2024-04-06T14:21:25.837 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_flush": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragmentdir": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_inodestats": 0 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_log": { 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "evadd": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "evex": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "evtrm": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "ev": 7, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "evexg": 0, 2024-04-06T14:21:25.838 INFO:teuthology.orchestra.run.smithi134.stdout: "evexd": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "segadd": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "segex": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "segtrm": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "seg": 1, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "segexg": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "segexd": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "expos": 4194304, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "wrpos": 4196619, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "rdpos": 4196478, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "jlat": { 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.839 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "replayed": 7 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_mem": { 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "ino": 3, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "ino+": 3, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "ino-": 0, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dir": 1, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dir+": 1, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dir-": 0, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dn": 0, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dn+": 0, 2024-04-06T14:21:25.840 INFO:teuthology.orchestra.run.smithi134.stdout: "dn-": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "cap": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "cap+": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "cap-": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "rss": 47880, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "heap": 207156 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_server": { 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_client_request": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_request": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_session": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_peer_request": 0, 2024-04-06T14:21:25.842 INFO:teuthology.orchestra.run.smithi134.stdout: "req_create_latency": { 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getattr_latency": { 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.843 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "req_link_latency": { 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookup_latency": { 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.844 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupino_latency": { 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupname_latency": { 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.845 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.846 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lssnap_latency": { 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mkdir_latency": { 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mknod_latency": { 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.847 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mksnap_latency": { 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "req_open_latency": { 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "req_readdir_latency": { 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.848 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rename_latency": { 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.849 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmdir_latency": { 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.850 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setattr_latency": { 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.851 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setlayout_latency": { 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setxattr_latency": { 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.852 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "req_symlink_latency": { 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "req_unlink_latency": { 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:25.853 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_sessions": { 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "session_count": 2, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "session_add": 4, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "session_remove": 2, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_open": 3, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_stale": 0, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "total_load": 0, 2024-04-06T14:21:25.854 INFO:teuthology.orchestra.run.smithi134.stdout: "average_load": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "avg_session_uptime": 5102, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "mempool": { 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:25.855 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:25.856 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:25.857 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_bytes": 189433, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_items": 52, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_bytes": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_items": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_items": 188, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:25.858 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_items": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_bytes": 8320, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_items": 13, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_items": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_items": 0 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "objecter": { 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "op_active": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "op_laggy": 0, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send": 3006, 2024-04-06T14:21:25.859 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send_bytes": 21, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_resend": 3, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_reply": 3006, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_latency": { 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 3006, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 16.561622353, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.005509521 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_inflight": 0, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "oplen_avg": { 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 3006, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 3007 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op": 3006, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_r": 3006, 2024-04-06T14:21:25.860 INFO:teuthology.orchestra.run.smithi134.stdout: "op_w": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "op_rmw": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "op_pg": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_stat": 1971, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_create": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_read": 1033, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_write": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writefull": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writesame": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_append": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_zero": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_truncate": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_delete": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_mapext": 0, 2024-04-06T14:21:25.861 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_getxattr": 1, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_setxattr": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_call": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_watch": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_notify": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_other": 0, 2024-04-06T14:21:25.862 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_active": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_send": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_resend": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_ping": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_active": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_send": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_resend": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_active": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_send": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_resend": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_active": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_send": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_resend": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "command_active": 0, 2024-04-06T14:21:25.863 INFO:teuthology.orchestra.run.smithi134.stdout: "command_send": 0, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "command_resend": 0, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "map_epoch": 104, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "map_full": 0, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "map_inc": 23, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_sessions": 5, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_open": 6, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_close": 1, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_laggy": 0, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_wr": 0, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_rd": 2, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_del": 0 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "oft": { 2024-04-06T14:21:25.864 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_objs": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_updates": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_removes": 0 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "osd-slow-ops": { 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_ops_count": 0 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "purge_queue": { 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops_high_water": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_high_water": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executed": 0, 2024-04-06T14:21:25.865 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 3331, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 2011078, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 3331, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3331, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 2011078, 2024-04-06T14:21:25.866 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 3006, 2024-04-06T14:21:25.867 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 2320, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3006, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 2320, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 1024, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:25.868 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 3006, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 3006, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3006, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 3006, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:25.869 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.870 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle-0x55813b7f65a0": { 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 0, 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:25.871 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:25.872 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:25.872 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:25.872 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.872 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:25.872 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-04-06T14:21:26.231 INFO:tasks.check_counter:Daemon mds.f mds.exported=0 2024-04-06T14:21:26.231 INFO:tasks.check_counter:Daemon mds.f mds.imported=0 2024-04-06T14:21:26.231 INFO:tasks.check_counter:Daemon mds.f mds_cache.dir_update=0 2024-04-06T14:21:26.231 INFO:tasks.check_counter:Daemon mds.f mds_cache.dir_update_receipt=0 2024-04-06T14:21:26.231 INFO:tasks.check_counter:Daemon mds.f mds.root_rsnaps=0 2024-04-06T14:21:26.232 INFO:tasks.check_counter:Daemon mds.f mds_server.req_mksnap_latency.avgcount=0 2024-04-06T14:21:26.232 INFO:tasks.check_counter:Daemon mds.f mds_server.req_rmsnap_latency.avgcount=0 2024-04-06T14:21:26.232 DEBUG:tasks.check_counter:Getting stats from i 2024-04-06T14:21:26.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.i perf dump 2024-04-06T14:21:26.457 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:21:27.408 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 2598, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 1541, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 852990, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 658476, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 14, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.142852418, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.048667467, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.111045596, 2024-04-06T14:21:27.409 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.006419937, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1542, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.275735666, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000178816 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 3940, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000794656, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000201 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 3040, 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 448 2024-04-06T14:21:27.410 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 641, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 873, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 1570556, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 761318, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 21, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 7, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.093793598, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.032390501, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.064330523, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.007131308, 2024-04-06T14:21:27.411 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 870, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.130606685, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000150122 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 579, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000113907, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000196 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 1566032, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 645264 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:27.412 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 82, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 158, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 15657, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 548459, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 57, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 5, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.024405960, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.003713712, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.018850854, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.000184368, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 155, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.042727258, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000275659 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.413 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 147, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000024778, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000168 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 7072, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 134480 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "cct": { 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "total_workers": 1, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "unhealthy_workers": 0 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-MDSRank": { 2024-04-06T14:21:27.414 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 55, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.001577764, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000028686 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "finisher-PurgeQueue": { 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "queue_len": 0, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "complete_latency": { 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 2, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000008349, 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000004174 2024-04-06T14:21:27.415 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "mds": { 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "request": 259, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "reply": 7, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "reply_latency": { 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 7, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.001147472, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000163924 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_reply": 0, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "forward": 252, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_complete": 0, 2024-04-06T14:21:27.416 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_fetch_keys": 0, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_commit": 11, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_split": 0, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_merge": 0, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes": 33, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_top": 10, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_bottom": 23, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pin_tail": 0, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_pinned": 18, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_expired": 0, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "inodes_with_caps": 2, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "caps": 3, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "subtrees": 2, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse": 322, 2024-04-06T14:21:27.417 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_hit": 51, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_forward": 249, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_discover": 21, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_dir_fetch": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_remote_ino": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "traverse_lock": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "load_cent": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "q": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "exported": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "exported_inodes": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "imported": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "imported_inodes": 0, 2024-04-06T14:21:27.418 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_dir_fetch": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_backtrace_fetch": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "openino_peer_discover": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rfiles": 1038, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rbytes": 1742189200, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "root_rsnaps": 6, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_fetch": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_set_tag": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_backtrace_repaired": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_inotable_repaired": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_inodes": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dir_base_inodes": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_dirfrag_rstats": 0, 2024-04-06T14:21:27.419 INFO:teuthology.orchestra.run.smithi134.stdout: "scrub_file_inodes": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_inode_file_caps": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_revoke": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_grant": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_trunc": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flushsnap_ack": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "ceph_cap_op_flush_ack": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_caps_dirty": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_cap_release": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "process_request_cap_release": 0 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_cache": { 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update": 0, 2024-04-06T14:21:27.420 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_update_receipt": 103, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_try_discover": 62, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_send_discover": 61, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "dir_handle_discover": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_delayed": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_strays_enqueuing": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_created": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_enqueued": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_reintegrated": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "strays_migrated": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_processing": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_enqueued": 0, 2024-04-06T14:21:27.421 INFO:teuthology.orchestra.run.smithi134.stdout: "num_recovering_prioritized": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_started": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "recovery_completed": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_enqueue_scrub": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_exportdir": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_flush": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragmentdir": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_fragstats": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "ireq_inodestats": 0 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_log": { 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "evadd": 7, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "evex": 0, 2024-04-06T14:21:27.422 INFO:teuthology.orchestra.run.smithi134.stdout: "evtrm": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "ev": 7, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "evexg": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "evexd": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "segadd": 1, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "segex": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "segtrm": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "seg": 1, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "segexg": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "segexd": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "expos": 4194304, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "wrpos": 4196619, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "rdpos": 0, 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "jlat": { 2024-04-06T14:21:27.423 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 7, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.011620109, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.001660015 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "replayed": 0 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_mem": { 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "ino": 36, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "ino+": 36, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "ino-": 0, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "dir": 63, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "dir+": 65, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "dir-": 2, 2024-04-06T14:21:27.424 INFO:teuthology.orchestra.run.smithi134.stdout: "dn": 33, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "dn+": 33, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "dn-": 0, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "cap": 3, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "cap+": 5, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "cap-": 2, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "rss": 52776, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "heap": 198964 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_server": { 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_client_request": 259, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "dispatch_server_request": 0, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_request": 259, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_client_session": 33, 2024-04-06T14:21:27.425 INFO:teuthology.orchestra.run.smithi134.stdout: "handle_peer_request": 0, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "req_create_latency": { 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getattr_latency": { 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 4, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000753537, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000188384 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getfilelock_latency": { 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.426 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "req_link_latency": { 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookup_latency": { 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 1, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000185240, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000185240 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookuphash_latency": { 2024-04-06T14:21:27.427 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupino_latency": { 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 2, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000208695, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000104347 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupname_latency": { 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.428 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupparent_latency": { 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lookupsnap_latency": { 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "req_lssnap_latency": { 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.429 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mkdir_latency": { 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mknod_latency": { 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "req_mksnap_latency": { 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.430 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "req_open_latency": { 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "req_readdir_latency": { 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rename_latency": { 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.431 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "req_renamesnap_latency": { 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "req_snapdiff_latency": { 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmdir_latency": { 2024-04-06T14:21:27.432 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmsnap_latency": { 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "req_rmxattr_latency": { 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.433 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setattr_latency": { 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setdirlayout_latency": { 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setfilelock_latency": { 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.434 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setlayout_latency": { 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "req_setxattr_latency": { 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "req_symlink_latency": { 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.435 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "req_unlink_latency": { 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_revoke_eviction": 0, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "cap_acquisition_throttle": 0, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "req_getvxattr_latency": { 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.436 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_sessions": { 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "session_count": 2, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "session_add": 4, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "session_remove": 2, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_open": 2, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "sessions_stale": 0, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "total_load": 5, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "average_load": 1, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "avg_session_uptime": 938, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "mdthresh_evicted": 0 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.437 INFO:teuthology.orchestra.run.smithi134.stdout: "mempool": { 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:27.438 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:27.439 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_bytes": 274942, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_items": 134, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_bytes": 0, 2024-04-06T14:21:27.440 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_items": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_bytes": 8256, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_items": 188, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_items": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_bytes": 198296, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_items": 1119, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:27.441 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_items": 0, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_items": 0 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "objecter": { 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_active": 0, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_laggy": 0, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send": 47, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_send_bytes": 13058, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_resend": 33, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_reply": 47, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "op_latency": { 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 47, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.242994352, 2024-04-06T14:21:27.442 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.005170092 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op_inflight": 0, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "oplen_avg": { 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 47, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 72 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op": 47, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op_r": 0, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op_w": 47, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op_rmw": 0, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "op_pg": 0, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_stat": 0, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_create": 10, 2024-04-06T14:21:27.443 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_read": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_write": 7, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writefull": 8, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_writesame": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_append": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_zero": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_truncate": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_delete": 6, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_mapext": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_sparse_read": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_clonerange": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_getxattr": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_setxattr": 20, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_cmpxattr": 0, 2024-04-06T14:21:27.444 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_rmxattr": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_resetxattrs": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_call": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_watch": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_notify": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_src_cmpxattr": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_pgls_filter": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "osdop_other": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_active": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_send": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_resend": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "linger_ping": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_active": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_send": 0, 2024-04-06T14:21:27.445 INFO:teuthology.orchestra.run.smithi134.stdout: "poolop_resend": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_active": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_send": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "poolstat_resend": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_active": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_send": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "statfs_resend": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "command_active": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "command_send": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "command_resend": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "map_epoch": 104, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "map_full": 0, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "map_inc": 23, 2024-04-06T14:21:27.446 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_sessions": 10, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_open": 15, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_session_close": 5, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_laggy": 0, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_wr": 20, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_rd": 0, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_del": 1 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "oft": { 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_objs": 1, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_kv_pairs": 0, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_updates": 0, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "omap_total_removes": 0 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "osd-slow-ops": { 2024-04-06T14:21:27.447 INFO:teuthology.orchestra.run.smithi134.stdout: "slow_ops_count": 259 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "purge_queue": { 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_ops_high_water": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executing_high_water": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_executed": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "pq_item_in_journal": 0 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:27.448 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 3317, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 2106469, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 3317, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 3317, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 2106469, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.449 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_bytes": { 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 47, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 13058, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 47, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 13058, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.450 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-objecter_ops": { 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 1024, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 47, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 47, 2024-04-06T14:21:27.451 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 47, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 47, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle": { 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:27.452 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-write_buf_throttle-0x55fe1ef320a0": { 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 3758096384, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:27.453 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 7, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 2315, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 7, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 7, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 2315, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:27.454 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-04-06T14:21:27.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:27 smithi105 ceph-mon[46969]: pgmap v504: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 52 KiB/s wr, 8 op/s 2024-04-06T14:21:27.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:27 smithi090 ceph-mon[43420]: pgmap v504: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 52 KiB/s wr, 8 op/s 2024-04-06T14:21:27.549 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:27 smithi134 ceph-mon[47549]: pgmap v504: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 52 KiB/s wr, 8 op/s 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds.exported=0 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds.imported=0 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds_cache.dir_update=0 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds_cache.dir_update_receipt=103 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds.root_rsnaps=6 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds_server.req_mksnap_latency.avgcount=0 2024-04-06T14:21:27.772 INFO:tasks.check_counter:Daemon mds.i mds_server.req_rmsnap_latency.avgcount=0 2024-04-06T14:21:27.772 DEBUG:tasks.check_counter:Getting stats from l 2024-04-06T14:21:27.773 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:052adcadd0534eddb0d73c5afbc083a84cd9dc46 shell --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 -- ceph daemon mds.l perf dump 2024-04-06T14:21:27.997 INFO:teuthology.orchestra.run.smithi134.stderr:Inferring config /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/mon.c/config 2024-04-06T14:21:28.989 INFO:teuthology.orchestra.run.smithi134.stdout:{ 2024-04-06T14:21:28.989 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-0": { 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 0, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 2, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 0, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 0, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 4, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 0, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.001330221, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.000000000, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.000319171, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:28.990 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 0, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 0 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-1": { 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 6, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 221, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 3360, 2024-04-06T14:21:28.991 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 191296, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 5, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 3, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.012335746, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.007581857, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.002495343, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 219, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.075409219, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000344334 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:28.992 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 3360, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 191296 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "AsyncMessenger::Worker-2": { 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_messages": 296, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages": 286, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_bytes": 1444624, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_bytes": 163776, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_created_connections": 2, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_active_connections": 1, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_total_time": 0.046741243, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_send_time": 0.013938691, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_recv_time": 0.032923171, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_running_fast_dispatch_time": 0.006950730, 2024-04-06T14:21:28.993 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_messages_queue_lat": { 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 285, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.085144189, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000298751 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_handle_ack_lat": { 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_recv_encrypted_bytes": 1444624, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "msgr_send_encrypted_bytes": 163776 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.994 INFO:teuthology.orchestra.run.smithi134.stdout: "cct": { 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "total_workers": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "unhealthy_workers": 0 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "mempool": { 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_bytes": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bloom_filter_items": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_bytes": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_alloc_items": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_bytes": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_data_items": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_bytes": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_onode_items": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_bytes": 0, 2024-04-06T14:21:28.995 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_meta_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_other_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_cache_buffer_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_extent_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_blob_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_shared_blob_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_inline_bl_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_bytes": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_fsck_items": 0, 2024-04-06T14:21:28.996 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_txc_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_deferred_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluestore_writing_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_reader_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "bluefs_file_writer_items": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_bytes": 149057, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_anon_items": 32, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_bytes": 0, 2024-04-06T14:21:28.997 INFO:teuthology.orchestra.run.smithi134.stdout: "buffer_meta_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_mapbl_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osd_pglog_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "osdmap_mapping_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_bytes": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "pgmap_items": 0, 2024-04-06T14:21:28.998 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_bytes": 928, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "mds_co_items": 2, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_bytes": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_1_items": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_bytes": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "unittest_2_items": 0 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: }, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "throttle-msgr_dispatch_throttler-mds": { 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "val": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "max": 104857600, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "get_started": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "get": 298, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "get_sum": 1402803, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_fail": 0, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "get_or_fail_success": 298, 2024-04-06T14:21:28.999 INFO:teuthology.orchestra.run.smithi134.stdout: "take": 0, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "take_sum": 0, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "put": 298, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "put_sum": 1402803, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "wait": { 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "avgcount": 0, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "sum": 0.000000000, 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: "avgtime": 0.000000000 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout: } 2024-04-06T14:21:29.000 INFO:teuthology.orchestra.run.smithi134.stdout:} 2024-04-06T14:21:29.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:29 smithi134 ceph-mon[47549]: pgmap v505: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 53 KiB/s wr, 10 op/s 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds_cache.dir_update_receipt' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.l 2024-04-06T14:21:29.397 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.l 2024-04-06T14:21:29.397 ERROR:teuthology.run_tasks:Manager failed: check-counter Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_d7d6e60867cadf19fbfd7655a551ae7d58060757/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/home/teuthworker/src/git.ceph.com_teuthology_d7d6e60867cadf19fbfd7655a551ae7d58060757/teuthology/task/__init__.py", line 132, in __exit__ self.end() File "/home/teuthworker/src/git.ceph.com_ceph_800ec6d64a7fa0648ad53edfc6e50b41fc4cd674/qa/tasks/check_counter.py", line 125, in end raise RuntimeError("The following counters failed to be set " RuntimeError: The following counters failed to be set on mds daemons: {'mds.exported', 'mds.imported'} 2024-04-06T14:21:29.398 DEBUG:teuthology.run_tasks:Unwinding manager kclient 2024-04-06T14:21:29.410 INFO:tasks.kclient:Unmounting kernel clients... 2024-04-06T14:21:29.410 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T14:21:29.410 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-06T14:21:29.445 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T14:21:29.446 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-06T14:21:29.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:29 smithi105 ceph-mon[46969]: pgmap v505: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 53 KiB/s wr, 10 op/s 2024-04-06T14:21:29.507 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:21:29.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:29 smithi090 ceph-mon[43420]: pgmap v505: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 30 KiB/s rd, 53 KiB/s wr, 10 op/s 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 -- 172.21.15.90:0/2071521664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc10d570 msgr2=0x7f8ffc10f960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 --2- 172.21.15.90:0/2071521664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc10d570 0x7f8ffc10f960 secure :-1 s=READY pgs=922 cs=0 l=1 rev1=1 crypto rx=0x7f8ff400b0a0 tx=0x7f8ff400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 -- 172.21.15.90:0/2071521664 shutdown_connections 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 --2- 172.21.15.90:0/2071521664 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc10d570 0x7f8ffc10f960 unknown :-1 s=CLOSED pgs=922 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 --2- 172.21.15.90:0/2071521664 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc0ffda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.680 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 --2- 172.21.15.90:0/2071521664 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 0x7f8ffc10d030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 -- 172.21.15.90:0/2071521664 >> 172.21.15.90:0/2071521664 conn(0x7f8ffc0fb5e0 msgr2=0x7f8ffc0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.676+0000 7f900332f700 1 -- 172.21.15.90:0/2071521664 shutdown_connections 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 -- 172.21.15.90:0/2071521664 wait complete. 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 Processor -- start 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 -- start start 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 0x7f8ffc1998f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc10d570 0x7f8ffc19dc40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ffc19e280 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ffc19e3f0 con 0x7f8ffc10d570 2024-04-06T14:21:29.681 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f900332f700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ffc19e5b0 con 0x7f8ffc100370 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.677+0000 7f90010cb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42374/0 (socket says 172.21.15.90:42374) 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 -- 172.21.15.90:0/2204674437 learned_addr learned my addr 172.21.15.90:0/2204674437 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90008ca700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 0x7f8ffc1998f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90018cc700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc10d570 0x7f8ffc19dc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 -- 172.21.15.90:0/2204674437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 msgr2=0x7f8ffc1998f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 0x7f8ffc1998f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 -- 172.21.15.90:0/2204674437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc10d570 msgr2=0x7f8ffc19dc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc10d570 0x7f8ffc19dc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 -- 172.21.15.90:0/2204674437 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ff4009d00 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.682 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f90010cb700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f8fec00c9b0 tx=0x7f8fec00cd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fec007b00 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8fec007c60 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fec018660 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8ffc19e890 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.678+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ffc1a60d0 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.683 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.679+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8ffc108790 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.680+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f8fec018880 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.681+0000 7f8ff27fc700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 0x7f8fe80a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:29.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.681+0000 7f90008ca700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 0x7f8fe80a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:29.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.681+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f8fec0cd090 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.684 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.681+0000 7f90008ca700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 0x7f8fe80a3750 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f8ff8005950 tx=0x7f8ff80058e0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:29.685 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.684+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f8fec0964f0 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.889 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.889+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8ffc069a60 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.890 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.890+0000 7f8ff27fc700 1 -- 172.21.15.90:0/2204674437 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v104) v1 ==== 74+0+23014 (secure 0 0 0) 0x7f8fec099b10 con 0x7f8ffc0ff9c0 2024-04-06T14:21:29.891 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:29.891 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":104,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:21:13.048510+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"103","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":103,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":14,"length":1},{"begin":27,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:21:29.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 msgr2=0x7f8fe80a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:29.893 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 0x7f8fe80a3750 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f8ff8005950 tx=0x7f8ff80058e0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 msgr2=0x7f8ffc1993b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 secure :-1 s=READY pgs=923 cs=0 l=1 rev1=1 crypto rx=0x7f8fec00c9b0 tx=0x7f8fec00cd70 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 shutdown_connections 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f8fe80a1290 0x7f8fe80a3750 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ffc10d570 0x7f8ffc19dc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f8ffc0ff9c0 0x7f8ffc1993b0 unknown :-1 s=CLOSED pgs=923 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 --2- 172.21.15.90:0/2204674437 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f8ffc100370 0x7f8ffc1998f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.892+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 >> 172.21.15.90:0/2204674437 conn(0x7f8ffc0fb5e0 msgr2=0x7f8ffc0fd9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.893+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 shutdown_connections 2024-04-06T14:21:29.894 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:29.893+0000 7f900332f700 1 -- 172.21.15.90:0/2204674437 wait complete. 2024-04-06T14:21:29.906 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-06T14:00:52.415568+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2024-04-06T14:03:59.243801+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '80', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2024-04-06T14:04:00.601881+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '103', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 27, 'snap_epoch': 103, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-06T14:21:29.906 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-06T14:21:30.034 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 -- 172.21.15.90:0/1467493653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c104f70 msgr2=0x7f0c2c10caf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 --2- 172.21.15.90:0/1467493653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c104f70 0x7f0c2c10caf0 secure :-1 s=READY pgs=924 cs=0 l=1 rev1=1 crypto rx=0x7f0c2400b0a0 tx=0x7f0c2400b3b0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 -- 172.21.15.90:0/1467493653 shutdown_connections 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 --2- 172.21.15.90:0/1467493653 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c104f70 0x7f0c2c10caf0 unknown :-1 s=CLOSED pgs=924 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 --2- 172.21.15.90:0/1467493653 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 --2- 172.21.15.90:0/1467493653 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c100820 0x7f0c2c104840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 -- 172.21.15.90:0/1467493653 >> 172.21.15.90:0/1467493653 conn(0x7f0c2c076d50 msgr2=0x7f0c2c077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 -- 172.21.15.90:0/1467493653 shutdown_connections 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.032+0000 7f0c34545700 1 -- 172.21.15.90:0/1467493653 wait complete. 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 Processor -- start 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 -- start start 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.035 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c104f70 0x7f0c2c197800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c19bba0 0x7f0c2c19c020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c19c620 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c19c790 con 0x7f0c2c104f70 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c34545700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c19c950 con 0x7f0c2c19bba0 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42404/0 (socket says 172.21.15.90:42404) 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 -- 172.21.15.90:0/2014722064 learned_addr learned my addr 172.21.15.90:0/2014722064 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c32ae2700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c19bba0 0x7f0c2c19c020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c31ae0700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c104f70 0x7f0c2c197800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 -- 172.21.15.90:0/2014722064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c19bba0 msgr2=0x7f0c2c19c020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c19bba0 0x7f0c2c19c020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 -- 172.21.15.90:0/2014722064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c104f70 msgr2=0x7f0c2c197800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c104f70 0x7f0c2c197800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.033+0000 7f0c322e1700 1 -- 172.21.15.90:0/2014722064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c24009d00 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.036 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c322e1700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7f0c1c00e9a0 tx=0x7f0c1c00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c1c009790 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c2c1a3eb0 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c1c010430 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c1c018680 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.037 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.034+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c2c1a4380 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.035+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c2c109250 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.036+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0c1c0105a0 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.037+0000 7f0c237fe700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 0x7f0c180a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.037+0000 7f0c31ae0700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 0x7f0c180a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.037+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0c1c014070 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.038 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.037+0000 7f0c31ae0700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 0x7f0c180a3750 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f0c2c198810 tx=0x7f0c28009260 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.040 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.039+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0c1c0964f0 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.235+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0c2c067b00 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.236 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.236+0000 7f0c237fe700 1 -- 172.21.15.90:0/2014722064 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v104) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0c1c026020 con 0x7f0c2c0ffe70 2024-04-06T14:21:30.236 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 1 2024-04-06T14:21:30.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 msgr2=0x7f0c180a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 0x7f0c180a3750 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f0c2c198810 tx=0x7f0c28009260 comp rx=0 tx=0).stop 2024-04-06T14:21:30.238 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 msgr2=0x7f0c2c1972c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 secure :-1 s=READY pgs=925 cs=0 l=1 rev1=1 crypto rx=0x7f0c1c00e9a0 tx=0x7f0c1c00ecb0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 shutdown_connections 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c180a1290 0x7f0c180a3750 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c19bba0 0x7f0c2c19c020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffe70 0x7f0c2c1972c0 unknown :-1 s=CLOSED pgs=925 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 --2- 172.21.15.90:0/2014722064 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c104f70 0x7f0c2c197800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 >> 172.21.15.90:0/2014722064 conn(0x7f0c2c076d50 msgr2=0x7f0c2c068120 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 shutdown_connections 2024-04-06T14:21:30.239 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.238+0000 7f0c34545700 1 -- 172.21.15.90:0/2014722064 wait complete. 2024-04-06T14:21:30.249 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:30 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2204674437' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:30.250 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2024-04-06T14:21:30.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 -- 172.21.15.90:0/2791971547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44100820 msgr2=0x7f5b44104840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/2791971547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44100820 0x7f5b44104840 secure :-1 s=READY pgs=926 cs=0 l=1 rev1=1 crypto rx=0x7f5b34009990 tx=0x7f5b34009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 -- 172.21.15.90:0/2791971547 shutdown_connections 2024-04-06T14:21:30.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/2791971547 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b44104f70 0x7f5b4410caf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.376 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/2791971547 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 0x7f5b44100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/2791971547 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44100820 0x7f5b44104840 unknown :-1 s=CLOSED pgs=926 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 -- 172.21.15.90:0/2791971547 >> 172.21.15.90:0/2791971547 conn(0x7f5b44076d50 msgr2=0x7f5b44077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 -- 172.21.15.90:0/2791971547 shutdown_connections 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.375+0000 7f5b49aa6700 1 -- 172.21.15.90:0/2791971547 wait complete. 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 Processor -- start 2024-04-06T14:21:30.377 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 -- start start 2024-04-06T14:21:30.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 0x7f5b44196060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b4419a940 0x7f5b4419adc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b4419b3c0 con 0x7f5b44104f70 2024-04-06T14:21:30.379 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b4419b530 con 0x7f5b440ffe70 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b49aa6700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b4419b6f0 con 0x7f5b4419a940 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b42ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b42ffd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42424/0 (socket says 172.21.15.90:42424) 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b42ffd700 1 -- 172.21.15.90:0/40032986 learned_addr learned my addr 172.21.15.90:0/40032986 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.376+0000 7f5b437fe700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 0x7f5b44196060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b43fff700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b4419a940 0x7f5b4419adc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 -- 172.21.15.90:0/40032986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b4419a940 msgr2=0x7f5b4419adc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b4419a940 0x7f5b4419adc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 -- 172.21.15.90:0/40032986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 msgr2=0x7f5b44196060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 0x7f5b44196060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 -- 172.21.15.90:0/40032986 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b340095f0 con 0x7f5b44104f70 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b42ffd700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 secure :-1 s=READY pgs=927 cs=0 l=1 rev1=1 crypto rx=0x7f5b34005770 tx=0x7f5b34009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b3401f070 con 0x7f5b44104f70 2024-04-06T14:21:30.380 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b34019cd0 con 0x7f5b44104f70 2024-04-06T14:21:30.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b44067e00 con 0x7f5b44104f70 2024-04-06T14:21:30.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b34024c40 con 0x7f5b44104f70 2024-04-06T14:21:30.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.377+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b440682b0 con 0x7f5b44104f70 2024-04-06T14:21:30.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.378+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5b44101320 con 0x7f5b44104f70 2024-04-06T14:21:30.381 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.379+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5b34019810 con 0x7f5b44104f70 2024-04-06T14:21:30.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.380+0000 7f5b40ff9700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 0x7f5b2c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.380+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5b340cfee0 con 0x7f5b44104f70 2024-04-06T14:21:30.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.380+0000 7f5b437fe700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 0x7f5b2c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.382 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.380+0000 7f5b437fe700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 0x7f5b2c0a3750 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f5b38005950 tx=0x7f5b3800b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.384 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.384+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5b3405fab0 con 0x7f5b44104f70 2024-04-06T14:21:30.392 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:30 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2204674437' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:30.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:30 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2204674437' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:30.582 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.582+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f5b44197360 con 0x7f5b44104f70 2024-04-06T14:21:30.583 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.582+0000 7f5b40ff9700 1 -- 172.21.15.90:0/40032986 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v104) v1 ==== 104+0+11 (secure 0 0 0) 0x7f5b340993a0 con 0x7f5b44104f70 2024-04-06T14:21:30.583 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 32 2024-04-06T14:21:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.584+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 msgr2=0x7f5b2c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.584+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 0x7f5b2c0a3750 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f5b38005950 tx=0x7f5b3800b410 comp rx=0 tx=0).stop 2024-04-06T14:21:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.584+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 msgr2=0x7f5b441965a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.584+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 secure :-1 s=READY pgs=927 cs=0 l=1 rev1=1 crypto rx=0x7f5b34005770 tx=0x7f5b34009580 comp rx=0 tx=0).stop 2024-04-06T14:21:30.585 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 shutdown_connections 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5b2c0a1290 0x7f5b2c0a3750 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5b4419a940 0x7f5b4419adc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b440ffe70 0x7f5b44196060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 --2- 172.21.15.90:0/40032986 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5b44104f70 0x7f5b441965a0 unknown :-1 s=CLOSED pgs=927 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 >> 172.21.15.90:0/40032986 conn(0x7f5b44076d50 msgr2=0x7f5b440fdd70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 shutdown_connections 2024-04-06T14:21:30.586 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.585+0000 7f5b49aa6700 1 -- 172.21.15.90:0/40032986 wait complete. 2024-04-06T14:21:30.597 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2024-04-06T14:21:30.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 -- 172.21.15.90:0/4000623506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34100820 msgr2=0x7fca34104850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 --2- 172.21.15.90:0/4000623506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34100820 0x7fca34104850 secure :-1 s=READY pgs=928 cs=0 l=1 rev1=1 crypto rx=0x7fca1c009990 tx=0x7fca1c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.725 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 -- 172.21.15.90:0/4000623506 shutdown_connections 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 --2- 172.21.15.90:0/4000623506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca34104f80 0x7fca3410cb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 --2- 172.21.15.90:0/4000623506 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34100250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.722+0000 7fca390f9700 1 --2- 172.21.15.90:0/4000623506 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34100820 0x7fca34104850 unknown :-1 s=CLOSED pgs=928 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.723+0000 7fca390f9700 1 -- 172.21.15.90:0/4000623506 >> 172.21.15.90:0/4000623506 conn(0x7fca34076d40 msgr2=0x7fca34077150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.723+0000 7fca390f9700 1 -- 172.21.15.90:0/4000623506 shutdown_connections 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.723+0000 7fca390f9700 1 -- 172.21.15.90:0/4000623506 wait complete. 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.723+0000 7fca390f9700 1 Processor -- start 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.723+0000 7fca390f9700 1 -- start start 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34195e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 0x7fca341a54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.726 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca341a5a00 con 0x7fca34104f80 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca341a5b40 con 0x7fca341968c0 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca341a5cd0 con 0x7fca340ffe70 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca32d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34195e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42454/0 (socket says 172.21.15.90:42454) 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca32d9d700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34195e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.134:3300/0 says I am v2:172.21.15.90:54368/0 (socket says 172.21.15.90:54368) 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 -- 172.21.15.90:0/1928598398 learned_addr learned my addr 172.21.15.90:0/1928598398 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3359e700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 0x7fca341a54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 -- 172.21.15.90:0/1928598398 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 msgr2=0x7fca34195e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34195e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 -- 172.21.15.90:0/1928598398 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 msgr2=0x7fca341a54c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 0x7fca341a54c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 -- 172.21.15.90:0/1928598398 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca1c0095f0 con 0x7fca34104f80 2024-04-06T14:21:30.727 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3359e700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 0x7fca341a54c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca3259c700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7fca1c0059d0 tx=0x7fca1c009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca1c01d070 con 0x7fca34104f80 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fca1c00fb30 con 0x7fca34104f80 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca341a5f50 con 0x7fca34104f80 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.724+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca1c022bf0 con 0x7fca34104f80 2024-04-06T14:21:30.728 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.725+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca341a6310 con 0x7fca34104f80 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.725+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca34108250 con 0x7fca34104f80 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.727+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fca1c00fca0 con 0x7fca34104f80 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.727+0000 7fca2bfff700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 0x7fca200a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.727+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fca1c0ce650 con 0x7fca34104f80 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.727+0000 7fca32d9d700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 0x7fca200a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:30.729 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.728+0000 7fca32d9d700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 0x7fca200a3750 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fca24005950 tx=0x7fca2400b410 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:30.731 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.731+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fca1c097b10 con 0x7fca34104f80 2024-04-06T14:21:30.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.927+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fca340695c0 con 0x7fca34104f80 2024-04-06T14:21:30.928 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.928+0000 7fca2bfff700 1 -- 172.21.15.90:0/1928598398 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v104) v1 ==== 100+0+11 (secure 0 0 0) 0x7fca1c09b130 con 0x7fca34104f80 2024-04-06T14:21:30.928 INFO:teuthology.orchestra.run.smithi090.stdout:pg_num: 64 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 msgr2=0x7fca200a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 0x7fca200a3750 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fca24005950 tx=0x7fca2400b410 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 msgr2=0x7fca34196380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 secure :-1 s=READY pgs=929 cs=0 l=1 rev1=1 crypto rx=0x7fca1c0059d0 tx=0x7fca1c009580 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 shutdown_connections 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fca200a1290 0x7fca200a3750 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca341968c0 0x7fca341a54c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fca340ffe70 0x7fca34195e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 --2- 172.21.15.90:0/1928598398 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fca34104f80 0x7fca34196380 unknown :-1 s=CLOSED pgs=929 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 >> 172.21.15.90:0/1928598398 conn(0x7fca34076d40 msgr2=0x7fca340fde80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 shutdown_connections 2024-04-06T14:21:30.931 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:30.930+0000 7fca390f9700 1 -- 172.21.15.90:0/1928598398 wait complete. 2024-04-06T14:21:30.943 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 -- 172.21.15.90:0/1662268961 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e0068aa0 msgr2=0x7f00e0105970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 --2- 172.21.15.90:0/1662268961 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e0068aa0 0x7f00e0105970 secure :-1 s=READY pgs=930 cs=0 l=1 rev1=1 crypto rx=0x7f00d00099c0 tx=0x7f00d0009cd0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 -- 172.21.15.90:0/1662268961 shutdown_connections 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 --2- 172.21.15.90:0/1662268961 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e01060a0 0x7f00e010dc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 --2- 172.21.15.90:0/1662268961 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e00680f0 0x7f00e00684d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 --2- 172.21.15.90:0/1662268961 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e0068aa0 0x7f00e0105970 unknown :-1 s=CLOSED pgs=930 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 -- 172.21.15.90:0/1662268961 >> 172.21.15.90:0/1662268961 conn(0x7f00e0076d50 msgr2=0x7f00e0077160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 -- 172.21.15.90:0/1662268961 shutdown_connections 2024-04-06T14:21:31.071 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.068+0000 7f00e73be700 1 -- 172.21.15.90:0/1662268961 wait complete. 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 Processor -- start 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 -- start start 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e01060a0 0x7f00e01976e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 0x7f00e019bf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e019c500 con 0x7f00e00680f0 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e019c670 con 0x7f00e01060a0 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e73be700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f00e019c830 con 0x7f00e019ba80 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42474/0 (socket says 172.21.15.90:42474) 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 -- 172.21.15.90:0/2877622680 learned_addr learned my addr 172.21.15.90:0/2877622680 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e4959700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e01060a0 0x7f00e01976e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:31.072 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e595b700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 0x7f00e019bf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 -- 172.21.15.90:0/2877622680 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 msgr2=0x7f00e019bf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 0x7f00e019bf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 -- 172.21.15.90:0/2877622680 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e01060a0 msgr2=0x7f00e01976e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e01060a0 0x7f00e01976e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.069+0000 7f00e515a700 1 -- 172.21.15.90:0/2877622680 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f00d00095f0 con 0x7f00e00680f0 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.070+0000 7f00e595b700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 0x7f00e019bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:31.073 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.070+0000 7f00e515a700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7f00dc00e9a0 tx=0x7f00dc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.071+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00dc009790 con 0x7f00e00680f0 2024-04-06T14:21:31.074 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.071+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f00e01a3d90 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.071+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f00dc00cbd0 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.071+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f00dc018680 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.071+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f00e01a4210 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.072+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f00e0109250 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.073+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f00dc00cd40 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.074+0000 7f00d67fc700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 0x7f00cc0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.074+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f00dc014070 con 0x7f00e00680f0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.074+0000 7f00e4959700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 0x7f00cc0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:31.075 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.074+0000 7f00e4959700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 0x7f00cc0a3750 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f00d0009990 tx=0x7f00d0005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:31.078 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.078+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f00dc052ab0 con 0x7f00e00680f0 2024-04-06T14:21:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:31 smithi090 ceph-mon[43420]: pgmap v506: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 42 KiB/s wr, 8 op/s 2024-04-06T14:21:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2014722064' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/40032986' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.288 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:31 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1928598398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.293+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f00e0102450 con 0x7f00e00680f0 2024-04-06T14:21:31.293 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.293+0000 7f00d67fc700 1 -- 172.21.15.90:0/2877622680 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 17 entries v104) v1 ==== 81+0+1000 (secure 0 0 0) 0x7f00dc0963a0 con 0x7f00e00680f0 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/2298955235 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/319323044 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/3768713918 2024-04-07T13:58:42.758062+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6800/1992032630 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6800/2246618981 2024-04-07T13:59:04.100187+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6800/1491440652 2024-04-07T13:58:42.758062+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/2609293956 2024-04-07T13:59:04.100187+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6801/1491440652 2024-04-07T13:58:42.758062+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/3659831004 2024-04-07T13:58:42.758062+0000 2024-04-06T14:21:31.294 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/3538889590 2024-04-07T13:58:42.758062+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6801/2246618981 2024-04-07T13:59:04.100187+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/1621883497 2024-04-07T13:59:04.100187+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/888041862 2024-04-07T13:59:04.100187+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:6801/1992032630 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/177425578 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/2318576826 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.295 INFO:teuthology.orchestra.run.smithi090.stdout:172.21.15.90:0/582164689 2024-04-07T14:05:23.551355+0000 2024-04-06T14:21:31.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.295+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 msgr2=0x7f00cc0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:31.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.295+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 0x7f00cc0a3750 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f00d0009990 tx=0x7f00d0005fd0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.296 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.295+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 msgr2=0x7f00e01971a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.295+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 secure :-1 s=READY pgs=931 cs=0 l=1 rev1=1 crypto rx=0x7f00dc00e9a0 tx=0x7f00dc00ecb0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 shutdown_connections 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f00cc0a1290 0x7f00cc0a3750 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f00e019ba80 0x7f00e019bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f00e00680f0 0x7f00e01971a0 unknown :-1 s=CLOSED pgs=931 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 --2- 172.21.15.90:0/2877622680 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f00e01060a0 0x7f00e01976e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 >> 172.21.15.90:0/2877622680 conn(0x7f00e0076d50 msgr2=0x7f00e00fe010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 shutdown_connections 2024-04-06T14:21:31.297 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:31.296+0000 7f00e73be700 1 -- 172.21.15.90:0/2877622680 wait complete. 2024-04-06T14:21:31.298 INFO:teuthology.orchestra.run.smithi090.stderr:listed 17 entries 2024-04-06T14:21:31.308 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2024-04-06T14:21:31.309 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.309 DEBUG:teuthology.orchestra.run.smithi090:> sudo umount /tmp/cephfs 2024-04-06T14:21:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:31 smithi134 ceph-mon[47549]: pgmap v506: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 42 KiB/s wr, 8 op/s 2024-04-06T14:21:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2014722064' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/40032986' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:31 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1928598398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:31 smithi105 ceph-mon[46969]: pgmap v506: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 42 KiB/s wr, 8 op/s 2024-04-06T14:21:31.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2014722064' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/40032986' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.499 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:31 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1928598398' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2024-04-06T14:21:31.610 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:21:31.610 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.610 DEBUG:teuthology.orchestra.run.smithi090:> (cd /home/ubuntu/cephtest && exec rmdir -- /tmp/cephfs) 2024-04-06T14:21:31.691 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.691 DEBUG:teuthology.orchestra.run.smithi090:> ip netns list 2024-04-06T14:21:31.733 INFO:teuthology.orchestra.run.smithi090.stdout:ceph-ns--tmp-cephfs (id: 0) 2024-04-06T14:21:31.733 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.733 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip netns delete ceph-ns--tmp-cephfs 2024-04-06T14:21:31.819 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.819 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link delete ceph-brx 2024-04-06T14:21:31.886 INFO:tasks.kclient:Unmounting kernel clients... 2024-04-06T14:21:31.886 DEBUG:teuthology.orchestra.run.smithi090:> set -ex 2024-04-06T14:21:31.887 DEBUG:teuthology.orchestra.run.smithi090:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-06T14:21:31.905 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=16244116k,nr_inodes=4061029,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0 cgroup /sys/fs/cgroup/systemd cgroup rw,seclabel,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0 cgroup /sys/fs/cgroup/hugetlb cgroup rw,seclabel,nosuid,nodev,noexec,relatime,hugetlb 0 0 cgroup /sys/fs/cgroup/devices cgroup rw,seclabel,nosuid,nodev,noexec,relatime,devices 0 0 cgroup /sys/fs/cgroup/memory cgroup rw,seclabel,nosuid,nodev,noexec,relatime,memory 0 0 cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,seclabel,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 cgroup /sys/fs/cgroup/freezer cgroup rw,seclabel,nosuid,nodev,noexec,relatime,freezer 0 0 cgroup /sys/fs/cgroup/blkio cgroup rw,seclabel,nosuid,nodev,noexec,relatime,blkio 0 0 cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,seclabel,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0 cgroup /sys/fs/cgroup/cpuset cgroup rw,seclabel,nosuid,nodev,noexec,relatime,cpuset 0 0 cgroup /sys/fs/cgroup/rdma cgroup rw,seclabel,nosuid,nodev,noexec,relatime,rdma 0 0 cgroup /sys/fs/cgroup/pids cgroup rw,seclabel,nosuid,nodev,noexec,relatime,pids 0 0 cgroup /sys/fs/cgroup/perf_event cgroup rw,seclabel,nosuid,nodev,noexec,relatime,perf_event 0 0 none /sys/kernel/tracing tracefs rw,seclabel,relatime 0 0 configfs /sys/kernel/config configfs rw,relatime 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=45,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=2854 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3252508k,mode=700,uid=1000,gid=1304 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/sda1 /var/lib/containers/storage/overlay ext4 rw,seclabel,relatime 0 0 overlay /var/lib/containers/storage/overlay/a58039155d1f1cf193bc6a59d4233c79990880c4654b35e14d88a32a14d3d383/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/a58039155d1f1cf193bc6a59d4233c79990880c4654b35e14d88a32a14d3d383/diff,workdir=/var/lib/containers/storage/overlay/a58039155d1f1cf193bc6a59d4233c79990880c4654b35e14d88a32a14d3d383/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/f8bf6d0954744e325945d87dc7f51a9c15e680e2a395bd40fd6cffd70ca05bf7/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/f8bf6d0954744e325945d87dc7f51a9c15e680e2a395bd40fd6cffd70ca05bf7/diff,workdir=/var/lib/containers/storage/overlay/f8bf6d0954744e325945d87dc7f51a9c15e680e2a395bd40fd6cffd70ca05bf7/work,metacopy=on,volatile 0 0 tmpfs /run/user/0 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3252508k,mode=700 0 0 binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0 overlay /var/lib/containers/storage/overlay/f79b17f40a682a51ee4d4091d278825a3710218719edbf7bbebddf408d6c3951/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/f79b17f40a682a51ee4d4091d278825a3710218719edbf7bbebddf408d6c3951/diff,workdir=/var/lib/containers/storage/overlay/f79b17f40a682a51ee4d4091d278825a3710218719edbf7bbebddf408d6c3951/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/50b9d1d58e7d6844bb759b461d5deb2f85bf0e94d5f844c039352e636fe561b5/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/50b9d1d58e7d6844bb759b461d5deb2f85bf0e94d5f844c039352e636fe561b5/diff,workdir=/var/lib/containers/storage/overlay/50b9d1d58e7d6844bb759b461d5deb2f85bf0e94d5f844c039352e636fe561b5/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/41f2231c7afec94526dc44c5bfc853cec6e9d555b61caa3008c0b8d3310ca9f1/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/41f2231c7afec94526dc44c5bfc853cec6e9d555b61caa3008c0b8d3310ca9f1/diff,workdir=/var/lib/containers/storage/overlay/41f2231c7afec94526dc44c5bfc853cec6e9d555b61caa3008c0b8d3310ca9f1/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/d6c8fca9938ed5596db3ffbd584bcc6a72ab3abedc1415617100a1de34b7add0/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/d6c8fca9938ed5596db3ffbd584bcc6a72ab3abedc1415617100a1de34b7add0/diff,workdir=/var/lib/containers/storage/overlay/d6c8fca9938ed5596db3ffbd584bcc6a72ab3abedc1415617100a1de34b7add0/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/182d8c21cbff040846348c2ca24881a5694e52ff1ded536236618ace00e6f9d2/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/182d8c21cbff040846348c2ca24881a5694e52ff1ded536236618ace00e6f9d2/diff,workdir=/var/lib/containers/storage/overlay/182d8c21cbff040846348c2ca24881a5694e52ff1ded536236618ace00e6f9d2/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/6a2929d3e36afd976bf618eed54d8f039440b798bd1559bc6059385f1657fd10/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/6a2929d3e36afd976bf618eed54d8f039440b798bd1559bc6059385f1657fd10/diff,workdir=/var/lib/containers/storage/overlay/6a2929d3e36afd976bf618eed54d8f039440b798bd1559bc6059385f1657fd10/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/3b3a0e0887b1d43cc378770f2095e203410d5cbd6cfcbeb64987b126bbc1daad/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/3b3a0e0887b1d43cc378770f2095e203410d5cbd6cfcbeb64987b126bbc1daad/diff,workdir=/var/lib/containers/storage/overlay/3b3a0e0887b1d43cc378770f2095e203410d5cbd6cfcbeb64987b126bbc1daad/work,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/a882d815dd509bc181d865eb1ed85697b8bff1e067952fc515da8dd27edd78b5/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/P72THDVEVTBXA2V6HG53BQ675U:/var/lib/containers/storage/overlay/l/CXOOVZ7G2SIPOV63CJVGB3AAXA,upperdir=/var/lib/containers/storage/overlay/a882d815dd509bc181d865eb1ed85697b8bff1e067952fc515da8dd27edd78b5/diff,workdir=/var/lib/containers/storage/overlay/a882d815dd509bc181d865eb1ed85697b8bff1e067952fc515da8dd27edd78b5/work,metacopy=on,volatile 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 2024-04-06T14:21:31.905 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.905 DEBUG:teuthology.orchestra.run.smithi090:> ip netns list 2024-04-06T14:21:31.963 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-06T14:21:31.963 DEBUG:teuthology.orchestra.run.smithi090:> sudo ip link delete ceph-brx 2024-04-06T14:21:32.122 INFO:teuthology.orchestra.run.smithi090.stderr:Cannot find device "ceph-brx" 2024-04-06T14:21:32.122 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:32 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2877622680' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-06T14:21:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:21:32.124 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-06T14:21:32.135 INFO:tasks.cephadm:Teardown begin 2024-04-06T14:21:32.135 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-04-06T14:21:32.135 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 -- 172.21.15.90:0/1814712937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30101820 msgr2=0x7f2a30105860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1814712937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30101820 0x7f2a30105860 secure :-1 s=READY pgs=932 cs=0 l=1 rev1=1 crypto rx=0x7f2a24009990 tx=0x7f2a24009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 -- 172.21.15.90:0/1814712937 shutdown_connections 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1814712937 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a3010db10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1814712937 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30100e70 0x7f2a30101250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1814712937 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30101820 0x7f2a30105860 unknown :-1 s=CLOSED pgs=932 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.261 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 -- 172.21.15.90:0/1814712937 >> 172.21.15.90:0/1814712937 conn(0x7f2a30078170 msgr2=0x7f2a30078580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 -- 172.21.15.90:0/1814712937 shutdown_connections 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.258+0000 7f2a37c71700 1 -- 172.21.15.90:0/1814712937 wait complete. 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 Processor -- start 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 -- start start 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a300735f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 0x7f2a301a51c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a301a5790 con 0x7f2a30100e70 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a301a5900 con 0x7f2a30105f90 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a37c71700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a301a5ac0 con 0x7f2a30073b30 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42496/0 (socket says 172.21.15.90:42496) 2024-04-06T14:21:32.262 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 -- 172.21.15.90:0/1975144009 learned_addr learned my addr 172.21.15.90:0/1975144009 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a3520c700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a300735f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 -- 172.21.15.90:0/1975144009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 msgr2=0x7f2a301a51c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a3620e700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 0x7f2a301a51c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 0x7f2a301a51c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 -- 172.21.15.90:0/1975144009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 msgr2=0x7f2a300735f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a300735f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a35a0d700 1 -- 172.21.15.90:0/1975144009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a240095f0 con 0x7f2a30100e70 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a3520c700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a300735f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.259+0000 7f2a3620e700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 0x7f2a301a51c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a35a0d700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 secure :-1 s=READY pgs=933 cs=0 l=1 rev1=1 crypto rx=0x7f2a2800c9b0 tx=0x7f2a2800cd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a28007b00 con 0x7f2a30100e70 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2a28007c60 con 0x7f2a30100e70 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a301a5da0 con 0x7f2a30100e70 2024-04-06T14:21:32.263 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2a28018660 con 0x7f2a30100e70 2024-04-06T14:21:32.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.260+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a301a6250 con 0x7f2a30100e70 2024-04-06T14:21:32.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.261+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a30109260 con 0x7f2a30100e70 2024-04-06T14:21:32.264 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.262+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f2a28018880 con 0x7f2a30100e70 2024-04-06T14:21:32.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.262+0000 7f2a2effd700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 0x7f2a180a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.262+0000 7f2a3520c700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 0x7f2a180a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.262+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f2a280ccd80 con 0x7f2a30100e70 2024-04-06T14:21:32.265 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.263+0000 7f2a3520c700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 0x7f2a180a3750 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f2a24005b40 tx=0x7f2a24005ab0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:32.267 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.267+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f2a280961b0 con 0x7f2a30100e70 2024-04-06T14:21:32.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:32 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2877622680' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-06T14:21:32.496 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:32 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2877622680' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-06T14:21:32.507 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.507+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f2a30074630 con 0x7f2a30100e70 2024-04-06T14:21:32.508 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.508+0000 7f2a2effd700 1 -- 172.21.15.90:0/1975144009 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f2a280997d0 con 0x7f2a30100e70 2024-04-06T14:21:32.508 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:32.509 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 msgr2=0x7f2a180a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 0x7f2a180a3750 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f2a24005b40 tx=0x7f2a24005ab0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 msgr2=0x7f2a300730b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 secure :-1 s=READY pgs=933 cs=0 l=1 rev1=1 crypto rx=0x7f2a2800c9b0 tx=0x7f2a2800cd70 comp rx=0 tx=0).stop 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 shutdown_connections 2024-04-06T14:21:32.511 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f2a180a1290 0x7f2a180a3750 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f2a30073b30 0x7f2a301a51c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f2a30100e70 0x7f2a300730b0 unknown :-1 s=CLOSED pgs=933 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 --2- 172.21.15.90:0/1975144009 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2a30105f90 0x7f2a300735f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 >> 172.21.15.90:0/1975144009 conn(0x7f2a30078170 msgr2=0x7f2a300fed20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 shutdown_connections 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.510+0000 7f2a37c71700 1 -- 172.21.15.90:0/1975144009 wait complete. 2024-04-06T14:21:32.512 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:21:32.524 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-06T14:21:32.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 -- 172.21.15.90:0/1127940729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c105b80 msgr2=0x7f5a6c105f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.648 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/1127940729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c105b80 0x7f5a6c105f60 secure :-1 s=READY pgs=934 cs=0 l=1 rev1=1 crypto rx=0x7f5a60009990 tx=0x7f5a60009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 -- 172.21.15.90:0/1127940729 shutdown_connections 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/1127940729 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c10c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/1127940729 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c105b80 0x7f5a6c105f60 unknown :-1 s=CLOSED pgs=934 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/1127940729 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c0ffae0 0x7f5a6c0fff60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 -- 172.21.15.90:0/1127940729 >> 172.21.15.90:0/1127940729 conn(0x7f5a6c0fb5f0 msgr2=0x7f5a6c0fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 -- 172.21.15.90:0/1127940729 shutdown_connections 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.645+0000 7f5a73b4e700 1 -- 172.21.15.90:0/1127940729 wait complete. 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 Processor -- start 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 -- start start 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c0ffae0 0x7f5a6c197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.649 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c105b80 0x7f5a6c1a2f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a6c198cb0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a6c1a34c0 con 0x7f5a6c0ffae0 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a73b4e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a6c1a3600 con 0x7f5a6c105b80 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a710e9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a710e9700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42508/0 (socket says 172.21.15.90:42508) 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a710e9700 1 -- 172.21.15.90:0/948618726 learned_addr learned my addr 172.21.15.90:0/948618726 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.646+0000 7f5a718ea700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c0ffae0 0x7f5a6c197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a720eb700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c105b80 0x7f5a6c1a2f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 -- 172.21.15.90:0/948618726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c105b80 msgr2=0x7f5a6c1a2f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.650 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c105b80 0x7f5a6c1a2f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 -- 172.21.15.90:0/948618726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c0ffae0 msgr2=0x7f5a6c197f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c0ffae0 0x7f5a6c197f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 -- 172.21.15.90:0/948618726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a600095f0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a710e9700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f5a6800d950 tx=0x7f5a6800dc60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a68009940 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5a68010460 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a6c1a3890 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5a6800f620 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.647+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a6c1a3da0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.651 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.648+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5a6c1080e0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.649+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f5a680108d0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.650+0000 7f5a5effd700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 0x7f5a580a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:32.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.650+0000 7f5a718ea700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 0x7f5a580a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:32.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.650+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f5a680ccbb0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.650+0000 7f5a718ea700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 0x7f5a580a3750 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f5a6000b5c0 tx=0x7f5a60005fd0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:32.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.654+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f5a68095fe0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.896 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.896+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5a6c1011e0 con 0x7f5a6c1004a0 2024-04-06T14:21:32.897 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.896+0000 7f5a5effd700 1 -- 172.21.15.90:0/948618726 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 24 v24) v1 ==== 94+0+10870 (secure 0 0 0) 0x7f5a68016090 con 0x7f5a6c1004a0 2024-04-06T14:21:32.898 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:32.898 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":24,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14565,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6835/1262397636","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6834","nonce":1262397636},{"type":"v1","addr":"172.21.15.90:6835","nonce":1262397636}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":14577,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6837/886535758","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6836","nonce":886535758},{"type":"v1","addr":"172.21.15.90:6837","nonce":886535758}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":14631,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6839/1327801998","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6838","nonce":1327801998},{"type":"v1","addr":"172.21.15.90:6839","nonce":1327801998}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":14643,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.90:6841/761413365","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6840","nonce":761413365},{"type":"v1","addr":"172.21.15.90:6841","nonce":761413365}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24434,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2124581639","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2124581639},{"type":"v1","addr":"172.21.15.105:6833","nonce":2124581639}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24439,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.134:6833/1837725694","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6832","nonce":1837725694},{"type":"v1","addr":"172.21.15.134:6833","nonce":1837725694}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":24,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-06T14:04:01.430508+0000","modified":"2024-04-06T14:05:24.555555+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":3,"in":[0,1,2],"up":{"mds_0":24515,"mds_1":24490,"mds_2":24485},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24457":{"gid":24457,"name":"f","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6835/2168599536","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6834","nonce":2168599536},{"type":"v1","addr":"172.21.15.134:6835","nonce":2168599536}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24458":{"gid":24458,"name":"e","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.105:6835/3686963284","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6834","nonce":3686963284},{"type":"v1","addr":"172.21.15.105:6835","nonce":3686963284}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24469":{"gid":24469,"name":"c","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.134:6837/659065731","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6836","nonce":659065731},{"type":"v1","addr":"172.21.15.134:6837","nonce":659065731}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24485":{"gid":24485,"name":"h","rank":2,"incarnation":21,"state":"up:active","state_seq":6,"addr":"172.21.15.105:6837/2851238848","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6836","nonce":2851238848},{"type":"v1","addr":"172.21.15.105:6837","nonce":2851238848}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24490":{"gid":24490,"name":"i","rank":1,"incarnation":19,"state":"up:active","state_seq":5,"addr":"172.21.15.134:6839/1913582469","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6838","nonce":1913582469},{"type":"v1","addr":"172.21.15.134:6839","nonce":1913582469}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24515":{"gid":24515,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.105:6839/2253165057","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6838","nonce":2253165057},{"type":"v1","addr":"172.21.15.105:6839","nonce":2253165057}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2024-04-06T14:21:32.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 msgr2=0x7f5a580a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 0x7f5a580a3750 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f5a6000b5c0 tx=0x7f5a60005fd0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 msgr2=0x7f5a6c198440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:32.899 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 secure :-1 s=READY pgs=935 cs=0 l=1 rev1=1 crypto rx=0x7f5a6800d950 tx=0x7f5a6800dc60 comp rx=0 tx=0).stop 2024-04-06T14:21:32.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 shutdown_connections 2024-04-06T14:21:32.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f5a580a1290 0x7f5a580a3750 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.901 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f5a6c105b80 0x7f5a6c1a2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5a6c0ffae0 0x7f5a6c197f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 --2- 172.21.15.90:0/948618726 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f5a6c1004a0 0x7f5a6c198440 unknown :-1 s=CLOSED pgs=935 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 >> 172.21.15.90:0/948618726 conn(0x7f5a6c0fb5f0 msgr2=0x7f5a6c0fbf00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 shutdown_connections 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:32.899+0000 7f5a73b4e700 1 -- 172.21.15.90:0/948618726 wait complete. 2024-04-06T14:21:32.902 INFO:teuthology.orchestra.run.smithi090.stderr:dumped fsmap epoch 24 2024-04-06T14:21:32.912 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 -- 172.21.15.90:0/1502401245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 msgr2=0x7f0c2c0fff60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 --2- 172.21.15.90:0/1502401245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c0fff60 secure :-1 s=READY pgs=936 cs=0 l=1 rev1=1 crypto rx=0x7f0c1c009990 tx=0x7f0c1c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 -- 172.21.15.90:0/1502401245 shutdown_connections 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 --2- 172.21.15.90:0/1502401245 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c10c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 --2- 172.21.15.90:0/1502401245 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c105f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 --2- 172.21.15.90:0/1502401245 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c0fff60 unknown :-1 s=CLOSED pgs=936 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 -- 172.21.15.90:0/1502401245 >> 172.21.15.90:0/1502401245 conn(0x7f0c2c0fb5f0 msgr2=0x7f0c2c0fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 -- 172.21.15.90:0/1502401245 shutdown_connections 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 -- 172.21.15.90:0/1502401245 wait complete. 2024-04-06T14:21:33.041 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.038+0000 7f0c31756700 1 Processor -- start 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 -- start start 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c198440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c1a2f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c198c90 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c1a34c0 con 0x7f0c2c1004a0 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c31756700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c2c1a3600 con 0x7f0c2c105b80 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:42534/0 (socket says 172.21.15.90:42534) 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 -- 172.21.15.90:0/3847743218 learned_addr learned my addr 172.21.15.90:0/3847743218 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2a7fc700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c198440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2b7fe700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c1a2f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.042 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 -- 172.21.15.90:0/3847743218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 msgr2=0x7f0c2c1a2f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c1a2f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 -- 172.21.15.90:0/3847743218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 msgr2=0x7f0c2c198440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c198440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 -- 172.21.15.90:0/3847743218 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c1c0095f0 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2a7fc700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c198440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2b7fe700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c1a2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.039+0000 7f0c2affd700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 secure :-1 s=READY pgs=937 cs=0 l=1 rev1=1 crypto rx=0x7f0c2000d9b0 tx=0x7f0c2000dd70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.040+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c2000bb60 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.040+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c20007e80 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.040+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c2c1a3860 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.040+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c20004dd0 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.043 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.040+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c2c1a3d70 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.041+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c2c108100 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.042+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0c20004020 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.044 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.043+0000 7f0c13fff700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 0x7f0c140a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.043+0000 7f0c2a7fc700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 0x7f0c140a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.043+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7f0c200cd090 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.045 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.043+0000 7f0c2a7fc700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 0x7f0c140a3750 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f0c1c00b2e0 tx=0x7f0c1c005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.046 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.046+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0c200964a0 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.246 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.246+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f0c2c0683a0 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.248 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.248+0000 7f0c13fff700 1 -- 172.21.15.90:0/3847743218 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v104) v1 ==== 74+0+23014 (secure 0 0 0) 0x7f0c20016020 con 0x7f0c2c0ffae0 2024-04-06T14:21:33.248 INFO:teuthology.orchestra.run.smithi090.stdout: 2024-04-06T14:21:33.248 INFO:teuthology.orchestra.run.smithi090.stdout:{"epoch":104,"fsid":"949dca1a-f41d-11ee-b649-cb9ed24678a4","created":"2024-04-06T13:58:13.321674+0000","modified":"2024-04-06T14:21:13.048510+0000","last_up_change":"2024-04-06T14:03:25.968704+0000","last_in_change":"2024-04-06T14:03:09.831484+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-06T14:00:52.415568+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-04-06T14:03:59.243801+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"80","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-04-06T14:04:00.601881+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"103","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":103,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c3c2803b-07fd-4e23-b926-5e9dcc735ce7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6802","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6803","nonce":1162173917}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6804","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6805","nonce":1162173917}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6808","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6809","nonce":1162173917}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6806","nonce":1162173917},{"type":"v1","addr":"172.21.15.90:6807","nonce":1162173917}]},"public_addr":"172.21.15.90:6803/1162173917","cluster_addr":"172.21.15.90:6805/1162173917","heartbeat_back_addr":"172.21.15.90:6809/1162173917","heartbeat_front_addr":"172.21.15.90:6807/1162173917","state":["exists","up"]},{"osd":1,"uuid":"efe81cdf-a49b-44a5-aece-bef13902a718","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6801","nonce":1321301965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6803","nonce":1321301965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6807","nonce":1321301965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1321301965},{"type":"v1","addr":"172.21.15.105:6805","nonce":1321301965}]},"public_addr":"172.21.15.105:6801/1321301965","cluster_addr":"172.21.15.105:6803/1321301965","heartbeat_back_addr":"172.21.15.105:6807/1321301965","heartbeat_front_addr":"172.21.15.105:6805/1321301965","state":["exists","up"]},{"osd":2,"uuid":"bb0a1674-931b-4a33-80b7-05b28a2b8f86","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6800","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6801","nonce":1139095317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6802","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6803","nonce":1139095317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6806","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6807","nonce":1139095317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6804","nonce":1139095317},{"type":"v1","addr":"172.21.15.134:6805","nonce":1139095317}]},"public_addr":"172.21.15.134:6801/1139095317","cluster_addr":"172.21.15.134:6803/1139095317","heartbeat_back_addr":"172.21.15.134:6807/1139095317","heartbeat_front_addr":"172.21.15.134:6805/1139095317","state":["exists","up"]},{"osd":3,"uuid":"b2f41b24-c336-46cd-af3a-f3cf7d83e3cf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6810","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6811","nonce":711321384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6812","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6813","nonce":711321384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6816","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6817","nonce":711321384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6814","nonce":711321384},{"type":"v1","addr":"172.21.15.90:6815","nonce":711321384}]},"public_addr":"172.21.15.90:6811/711321384","cluster_addr":"172.21.15.90:6813/711321384","heartbeat_back_addr":"172.21.15.90:6817/711321384","heartbeat_front_addr":"172.21.15.90:6815/711321384","state":["exists","up"]},{"osd":4,"uuid":"6bb3d511-2b64-45cb-845e-57589ef3fb59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6809","nonce":4184734218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6811","nonce":4184734218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6815","nonce":4184734218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":4184734218},{"type":"v1","addr":"172.21.15.105:6813","nonce":4184734218}]},"public_addr":"172.21.15.105:6809/4184734218","cluster_addr":"172.21.15.105:6811/4184734218","heartbeat_back_addr":"172.21.15.105:6815/4184734218","heartbeat_front_addr":"172.21.15.105:6813/4184734218","state":["exists","up"]},{"osd":5,"uuid":"989a2709-b8f9-47e7-9385-54bc5d4287ac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6808","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6809","nonce":1329717087}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6810","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6811","nonce":1329717087}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6814","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6815","nonce":1329717087}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6812","nonce":1329717087},{"type":"v1","addr":"172.21.15.134:6813","nonce":1329717087}]},"public_addr":"172.21.15.134:6809/1329717087","cluster_addr":"172.21.15.134:6811/1329717087","heartbeat_back_addr":"172.21.15.134:6815/1329717087","heartbeat_front_addr":"172.21.15.134:6813/1329717087","state":["exists","up"]},{"osd":6,"uuid":"5fe9a9e4-959e-4fd9-b50d-701606e38f79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6818","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6819","nonce":1241504412}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6820","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6821","nonce":1241504412}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6824","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6825","nonce":1241504412}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6822","nonce":1241504412},{"type":"v1","addr":"172.21.15.90:6823","nonce":1241504412}]},"public_addr":"172.21.15.90:6819/1241504412","cluster_addr":"172.21.15.90:6821/1241504412","heartbeat_back_addr":"172.21.15.90:6825/1241504412","heartbeat_front_addr":"172.21.15.90:6823/1241504412","state":["exists","up"]},{"osd":7,"uuid":"eef257f2-5d81-4b97-9186-03b54813ccea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":49,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6817","nonce":4248271964}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6819","nonce":4248271964}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6823","nonce":4248271964}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":4248271964},{"type":"v1","addr":"172.21.15.105:6821","nonce":4248271964}]},"public_addr":"172.21.15.105:6817/4248271964","cluster_addr":"172.21.15.105:6819/4248271964","heartbeat_back_addr":"172.21.15.105:6823/4248271964","heartbeat_front_addr":"172.21.15.105:6821/4248271964","state":["exists","up"]},{"osd":8,"uuid":"98c09ef0-8533-4ab9-b4ba-51aeb43ebc17","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6816","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6817","nonce":549202076}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6818","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6819","nonce":549202076}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6822","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6823","nonce":549202076}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6820","nonce":549202076},{"type":"v1","addr":"172.21.15.134:6821","nonce":549202076}]},"public_addr":"172.21.15.134:6817/549202076","cluster_addr":"172.21.15.134:6819/549202076","heartbeat_back_addr":"172.21.15.134:6823/549202076","heartbeat_front_addr":"172.21.15.134:6821/549202076","state":["exists","up"]},{"osd":9,"uuid":"599f84b9-90b8-4225-809c-96218058b7c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":63,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6826","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6827","nonce":154728526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6828","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6829","nonce":154728526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6832","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6833","nonce":154728526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.90:6830","nonce":154728526},{"type":"v1","addr":"172.21.15.90:6831","nonce":154728526}]},"public_addr":"172.21.15.90:6827/154728526","cluster_addr":"172.21.15.90:6829/154728526","heartbeat_back_addr":"172.21.15.90:6833/154728526","heartbeat_front_addr":"172.21.15.90:6831/154728526","state":["exists","up"]},{"osd":10,"uuid":"86cce4f5-4741-4f2b-87e5-0af0d083ae8e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":68,"up_thru":79,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6825","nonce":394238659}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6827","nonce":394238659}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6831","nonce":394238659}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":394238659},{"type":"v1","addr":"172.21.15.105:6829","nonce":394238659}]},"public_addr":"172.21.15.105:6825/394238659","cluster_addr":"172.21.15.105:6827/394238659","heartbeat_back_addr":"172.21.15.105:6831/394238659","heartbeat_front_addr":"172.21.15.105:6829/394238659","state":["exists","up"]},{"osd":11,"uuid":"1942c563-7632-4c97-b0a2-7f079c532b07","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":76,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6824","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6825","nonce":1707163828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6826","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6827","nonce":1707163828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6830","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6831","nonce":1707163828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.134:6828","nonce":1707163828},{"type":"v1","addr":"172.21.15.134:6829","nonce":1707163828}]},"public_addr":"172.21.15.134:6825/1707163828","cluster_addr":"172.21.15.134:6827/1707163828","heartbeat_back_addr":"172.21.15.134:6831/1707163828","heartbeat_front_addr":"172.21.15.134:6829/1707163828","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:15.828078+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:32.823297+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:00:49.618982+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:06.743542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:24.181079+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:40.947960+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:01:58.043667+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:15.263628+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:32.189910+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:02:49.412730+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:07.102656+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-06T14:03:24.208007+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.a","mappings":[{"from":2,"to":11}]},{"pgid":"2.1f","mappings":[{"from":8,"to":4}]},{"pgid":"3.1","mappings":[{"from":4,"to":9}]},{"pgid":"3.7","mappings":[{"from":3,"to":9}]},{"pgid":"3.e","mappings":[{"from":4,"to":6}]},{"pgid":"3.1d","mappings":[{"from":4,"to":8}]},{"pgid":"3.20","mappings":[{"from":5,"to":9}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.90:0/2298955235":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/319323044":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/3768713918":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6800/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:6800/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6800/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/2609293956":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1491440652":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3659831004":"2024-04-07T13:58:42.758062+0000","172.21.15.90:0/3538889590":"2024-04-07T13:58:42.758062+0000","172.21.15.90:6801/2246618981":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/1621883497":"2024-04-07T13:59:04.100187+0000","172.21.15.90:0/888041862":"2024-04-07T13:59:04.100187+0000","172.21.15.90:6801/1992032630":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/177425578":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/2318576826":"2024-04-07T14:05:23.551355+0000","172.21.15.90:0/582164689":"2024-04-07T14:05:23.551355+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":14,"length":1},{"begin":27,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-06T14:21:33.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 msgr2=0x7f0c140a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 0x7f0c140a3750 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f0c1c00b2e0 tx=0x7f0c1c005fb0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.250 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 msgr2=0x7f0c2c197f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 secure :-1 s=READY pgs=937 cs=0 l=1 rev1=1 crypto rx=0x7f0c2000d9b0 tx=0x7f0c2000dd70 comp rx=0 tx=0).stop 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 shutdown_connections 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f0c140a1290 0x7f0c140a3750 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0c2c105b80 0x7f0c2c1a2f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f0c2c0ffae0 0x7f0c2c197f00 unknown :-1 s=CLOSED pgs=937 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 --2- 172.21.15.90:0/3847743218 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c2c1004a0 0x7f0c2c198440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 >> 172.21.15.90:0/3847743218 conn(0x7f0c2c0fb5f0 msgr2=0x7f0c2c0fbe50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 shutdown_connections 2024-04-06T14:21:33.251 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.250+0000 7f0c31756700 1 -- 172.21.15.90:0/3847743218 wait complete. 2024-04-06T14:21:33.262 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:33 smithi090 ceph-mon[43420]: pgmap v507: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 31 KiB/s wr, 9 op/s 2024-04-06T14:21:33.262 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:33 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1975144009' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.262 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:33 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/948618726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.264 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2024-04-06T14:21:33.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 -- 172.21.15.90:0/649095128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4100f50 msgr2=0x7fbec4101330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.387 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 --2- 172.21.15.90:0/649095128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4100f50 0x7fbec4101330 secure :-1 s=READY pgs=938 cs=0 l=1 rev1=1 crypto rx=0x7fbeb4009990 tx=0x7fbeb4009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 -- 172.21.15.90:0/649095128 shutdown_connections 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 --2- 172.21.15.90:0/649095128 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec4106070 0x7fbec410dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 --2- 172.21.15.90:0/649095128 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4100f50 0x7fbec4101330 unknown :-1 s=CLOSED pgs=938 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 --2- 172.21.15.90:0/649095128 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4101900 0x7fbec4105940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 -- 172.21.15.90:0/649095128 >> 172.21.15.90:0/649095128 conn(0x7fbec40fc7f0 msgr2=0x7fbec40fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 -- 172.21.15.90:0/649095128 shutdown_connections 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.386+0000 7fbec9507700 1 -- 172.21.15.90:0/649095128 wait complete. 2024-04-06T14:21:33.388 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.387+0000 7fbec9507700 1 Processor -- start 2024-04-06T14:21:33.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.387+0000 7fbec9507700 1 -- start start 2024-04-06T14:21:33.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4100f50 0x7fbec41971f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.391 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec419bad0 0x7fbec419bf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbec419c550 con 0x7fbec4106070 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbec419c6c0 con 0x7fbec419bad0 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec9507700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbec419c880 con 0x7fbec4100f50 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60502/0 (socket says 172.21.15.90:60502) 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 -- 172.21.15.90:0/2654900295 learned_addr learned my addr 172.21.15.90:0/2654900295 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec2ffd700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4100f50 0x7fbec41971f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec37fe700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec419bad0 0x7fbec419bf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 -- 172.21.15.90:0/2654900295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4100f50 msgr2=0x7fbec41971f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4100f50 0x7fbec41971f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.392 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 -- 172.21.15.90:0/2654900295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec419bad0 msgr2=0x7fbec419bf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec419bad0 0x7fbec419bf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 -- 172.21.15.90:0/2654900295 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbeb40095f0 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbec27fc700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 secure :-1 s=READY pgs=939 cs=0 l=1 rev1=1 crypto rx=0x7fbeb800e970 tx=0x7fbeb800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbeb8009790 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.388+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbeb800eec0 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.389+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbec41a3de0 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.389+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbeb8005590 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.389+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbec41a4270 con 0x7fbec4106070 2024-04-06T14:21:33.393 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.390+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbec41092e0 con 0x7fbec4106070 2024-04-06T14:21:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mon[47549]: pgmap v507: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 31 KiB/s wr, 9 op/s 2024-04-06T14:21:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1975144009' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/948618726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.391+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fbeb800cbd0 con 0x7fbec4106070 2024-04-06T14:21:33.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.391+0000 7fbeabfff700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 0x7fbeac0a37d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.391+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(104..104 src has 1..104) v4 ==== 8556+0+0 (secure 0 0 0) 0x7fbeb8014070 con 0x7fbec4106070 2024-04-06T14:21:33.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.391+0000 7fbec2ffd700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 0x7fbeac0a37d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.395 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.392+0000 7fbec2ffd700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 0x7fbeac0a37d0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fbeb400b2e0 tx=0x7fbeb4005960 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.396 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.396+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fbeb80962d0 con 0x7fbec4106070 2024-04-06T14:21:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-mon[46969]: pgmap v507: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 31 KiB/s wr, 9 op/s 2024-04-06T14:21:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1975144009' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/948618726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-06T14:21:33.641 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.640+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fbec41984f0 con 0x7fbec4106070 2024-04-06T14:21:33.652 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.651+0000 7fbeabfff700 1 -- 172.21.15.90:0/2654900295 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v25) v1 ==== 161+0+0 (secure 0 0 0) 0x7fbeb8004ef0 con 0x7fbec4106070 2024-04-06T14:21:33.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.653+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 msgr2=0x7fbeac0a37d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 0x7fbeac0a37d0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fbeb400b2e0 tx=0x7fbeb4005960 comp rx=0 tx=0).stop 2024-04-06T14:21:33.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 msgr2=0x7fbec4197730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 secure :-1 s=READY pgs=939 cs=0 l=1 rev1=1 crypto rx=0x7fbeb800e970 tx=0x7fbeb800ec80 comp rx=0 tx=0).stop 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 shutdown_connections 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fbeac0a1310 0x7fbeac0a37d0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbec419bad0 0x7fbec419bf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fbec4100f50 0x7fbec41971f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 --2- 172.21.15.90:0/2654900295 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fbec4106070 0x7fbec4197730 unknown :-1 s=CLOSED pgs=939 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 >> 172.21.15.90:0/2654900295 conn(0x7fbec40fc7f0 msgr2=0x7fbec40fdbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 shutdown_connections 2024-04-06T14:21:33.655 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.654+0000 7fbec9507700 1 -- 172.21.15.90:0/2654900295 wait complete. 2024-04-06T14:21:33.656 INFO:teuthology.orchestra.run.smithi090.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-04-06T14:21:33.667 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2024-04-06T14:21:33.796 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 -- 172.21.15.90:0/1975292290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08381008c0 msgr2=0x7f083810cad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 --2- 172.21.15.90:0/1975292290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08381008c0 0x7f083810cad0 secure :-1 s=READY pgs=942 cs=0 l=1 rev1=1 crypto rx=0x7f083000b0d0 tx=0x7f083000b3e0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 -- 172.21.15.90:0/1975292290 shutdown_connections 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 --2- 172.21.15.90:0/1975292290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08381008c0 0x7f083810cad0 unknown :-1 s=CLOSED pgs=942 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 --2- 172.21.15.90:0/1975292290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f0838105da0 0x7f0838106180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 --2- 172.21.15.90:0/1975292290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08380fff00 0x7f0838100380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 -- 172.21.15.90:0/1975292290 >> 172.21.15.90:0/1975292290 conn(0x7f08380769b0 msgr2=0x7f0838076dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 -- 172.21.15.90:0/1975292290 shutdown_connections 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.793+0000 7f083fb20700 1 -- 172.21.15.90:0/1975292290 wait complete. 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 Processor -- start 2024-04-06T14:21:33.797 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 -- start start 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08381008c0 0x7f0838198690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 0x7f08381a31d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0838198ee0 con 0x7f08380fff00 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08381a3710 con 0x7f0838105da0 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083fb20700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08381a3850 con 0x7f08381008c0 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083d8bc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083d8bc700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60508/0 (socket says 172.21.15.90:60508) 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083d8bc700 1 -- 172.21.15.90:0/1462829521 learned_addr learned my addr 172.21.15.90:0/1462829521 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083d0bb700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08381008c0 0x7f0838198690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.794+0000 7f083e0bd700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 0x7f08381a31d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 -- 172.21.15.90:0/1462829521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08381008c0 msgr2=0x7f0838198690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.798 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08381008c0 0x7f0838198690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 -- 172.21.15.90:0/1462829521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 msgr2=0x7f08381a31d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 0x7f08381a31d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 -- 172.21.15.90:0/1462829521 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0830009d00 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083e0bd700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 0x7f08381a31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083d8bc700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f083400c9a0 tx=0x7f083400ccb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0834004220 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0834007d20 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08381a3ab0 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0834003c00 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.795+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08381a3fc0 con 0x7f08380fff00 2024-04-06T14:21:33.799 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.796+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08381081b0 con 0x7f08380fff00 2024-04-06T14:21:33.800 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.797+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f0834007860 con 0x7f08380fff00 2024-04-06T14:21:33.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.798+0000 7f082effd700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 0x7f08240a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:33.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.798+0000 7f083d0bb700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 0x7f08240a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:33.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.798+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 8770+0+0 (secure 0 0 0) 0x7f08340cc9b0 con 0x7f08380fff00 2024-04-06T14:21:33.801 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.798+0000 7f083d0bb700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 0x7f08240a3750 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f08380697b0 tx=0x7f0828009190 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:33.802 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:33.802+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f0834095d00 con 0x7f08380fff00 2024-04-06T14:21:33.997 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:33.997 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:33.997 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: starting mds.e at 2024-04-06T14:21:33.997 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-mds[84441]: -- 172.21.15.105:0/2178873006 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559f1d4421a0 con 0x559f1d419000 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: starting mds.b at 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-mds[84097]: -- 172.21.15.105:0/1133212779 <== mon.0 v2:172.21.15.90:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562b3cbe61a0 con 0x562b3cbbd000 2024-04-06T14:21:33.998 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:21:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: starting mds.h at 2024-04-06T14:21:34.038 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a[43397]: 2024-04-06T14:21:33.641+0000 7f91953a3700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-06T14:21:34.052 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.051+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f0838101370 con 0x7f08380fff00 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mds[85069]: -- 172.21.15.134:0/2967800434 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5620081f01a0 con 0x5620081c6c00 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: starting mds.c at 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mds[85412]: -- 172.21.15.134:0/1983975970 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559bef1801a0 con 0x559bef157000 2024-04-06T14:21:34.080 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: starting mds.i at 2024-04-06T14:21:34.081 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: ignoring --setuser ceph since I am not root 2024-04-06T14:21:34.081 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: ignoring --setgroup ceph since I am not root 2024-04-06T14:21:34.081 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-mds[84719]: -- 172.21.15.134:0/2598987929 <== mon.1 v2:172.21.15.134:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557be46841a0 con 0x557be465a800 2024-04-06T14:21:34.081 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:21:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: starting mds.f at 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/3847743218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: osdmap e105: 12 total, 12 up, 12 in 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-06T14:21:34.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:34 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/3847743218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: osdmap e105: 12 total, 12 up, 12 in 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-06T14:21:34.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:34 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/3847743218' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: osdmap e105: 12 total, 12 up, 12 in 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/2654900295' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-04-06T14:21:34.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: fsmap cephfs:0/3 6 up:standby, 3 failed 2024-04-06T14:21:34.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:34 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2024-04-06T14:21:34.654 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.654+0000 7f082effd700 1 -- 172.21.15.90:0/1462829521 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v26) v1 ==== 104+0+0 (secure 0 0 0) 0x7f0834099320 con 0x7f08380fff00 2024-04-06T14:21:34.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 msgr2=0x7f08240a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:34.657 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 0x7f08240a3750 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f08380697b0 tx=0x7f0828009190 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 msgr2=0x7f0838198150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 secure :-1 s=READY pgs=943 cs=0 l=1 rev1=1 crypto rx=0x7f083400c9a0 tx=0x7f083400ccb0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 shutdown_connections 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f08240a1290 0x7f08240a3750 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0838105da0 0x7f08381a31d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f08380fff00 0x7f0838198150 unknown :-1 s=CLOSED pgs=943 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 --2- 172.21.15.90:0/1462829521 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f08381008c0 0x7f0838198690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 >> 172.21.15.90:0/1462829521 conn(0x7f08380769b0 msgr2=0x7f08380feaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 shutdown_connections 2024-04-06T14:21:34.658 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.657+0000 7f083fb20700 1 -- 172.21.15.90:0/1462829521 wait complete. 2024-04-06T14:21:34.672 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 -- 172.21.15.90:0/2465046611 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 msgr2=0x7f6270105e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 --2- 172.21.15.90:0/2465046611 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f6270105e50 secure :-1 s=READY pgs=944 cs=0 l=1 rev1=1 crypto rx=0x7f626c009990 tx=0x7f626c009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 -- 172.21.15.90:0/2465046611 shutdown_connections 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 --2- 172.21.15.90:0/2465046611 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6270100420 0x7f627010c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 --2- 172.21.15.90:0/2465046611 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f6270105e50 unknown :-1 s=CLOSED pgs=944 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 --2- 172.21.15.90:0/2465046611 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f62700ffa60 0x7f62700ffee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 -- 172.21.15.90:0/2465046611 >> 172.21.15.90:0/2465046611 conn(0x7f62700fb570 msgr2=0x7f62700fd990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 -- 172.21.15.90:0/2465046611 shutdown_connections 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 -- 172.21.15.90:0/2465046611 wait complete. 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.801+0000 7f627719e700 1 Processor -- start 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- start start 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62700ffa60 0x7f6270196eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6270100420 0x7f62701973f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f627019bdd0 con 0x7f6270105a70 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f627019bf40 con 0x7f62700ffa60 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f627019c100 con 0x7f6270100420 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:34.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60520/0 (socket says 172.21.15.90:60520) 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 -- 172.21.15.90:0/1236862169 learned_addr learned my addr 172.21.15.90:0/1236862169 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627599b700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6270100420 0x7f62701973f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 -- 172.21.15.90:0/1236862169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6270100420 msgr2=0x7f62701973f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6270100420 0x7f62701973f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 -- 172.21.15.90:0/1236862169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62700ffa60 msgr2=0x7f6270196eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62700ffa60 0x7f6270196eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 -- 172.21.15.90:0/1236862169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f626c0095f0 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627699d700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 secure :-1 s=READY pgs=945 cs=0 l=1 rev1=1 crypto rx=0x7f6268007ee0 tx=0x7f626800d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6268017070 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f626800f040 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f627019c3e0 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6268005740 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.802+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62701a3c20 con 0x7f6270105a70 2024-04-06T14:21:34.806 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.803+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6270109020 con 0x7f6270105a70 2024-04-06T14:21:34.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.804+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7f62680058a0 con 0x7f6270105a70 2024-04-06T14:21:34.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.805+0000 7f62677fe700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 0x7f625c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:34.807 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.805+0000 7f627619c700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 0x7f625c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:34.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.805+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 8770+0+0 (secure 0 0 0) 0x7f62680ccc50 con 0x7f6270105a70 2024-04-06T14:21:34.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.805+0000 7f627619c700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 0x7f625c0a3750 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f626c00b5c0 tx=0x7f626c005fb0 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:34.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:34.809+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7f6268095fa0 con 0x7f6270105a70 2024-04-06T14:21:35.006 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.005+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f62701981b0 con 0x7f6270105a70 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: pgmap v509: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 266 KiB/s rd, 261 KiB/s wr, 8 op/s 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: Cluster is now healthy 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6837/442878082,v1:172.21.15.134:6838/442878082] up:boot 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6836/745783224,v1:172.21.15.134:6839/745783224] up:boot 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.134:6834/3757701937,v1:172.21.15.134:6835/3757701937] up:boot 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6834/1460644165,v1:172.21.15.105:6835/1460644165] up:boot 2024-04-06T14:21:35.393 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6836/151761867,v1:172.21.15.105:6837/151761867] up:boot 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: mds.? [v2:172.21.15.105:6838/2977723885,v1:172.21.15.105:6839/2977723885] up:boot 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: fsmap 12 up:standby 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:21:35.394 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:35 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: pgmap v509: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 266 KiB/s rd, 261 KiB/s wr, 8 op/s 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: Cluster is now healthy 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6837/442878082,v1:172.21.15.134:6838/442878082] up:boot 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6836/745783224,v1:172.21.15.134:6839/745783224] up:boot 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.134:6834/3757701937,v1:172.21.15.134:6835/3757701937] up:boot 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6834/1460644165,v1:172.21.15.105:6835/1460644165] up:boot 2024-04-06T14:21:35.497 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6836/151761867,v1:172.21.15.105:6837/151761867] up:boot 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: mds.? [v2:172.21.15.105:6838/2977723885,v1:172.21.15.105:6839/2977723885] up:boot 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: fsmap 12 up:standby 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:21:35.498 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:35 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: pgmap v509: 97 pgs: 97 active+clean; 5.2 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail; 266 KiB/s rd, 261 KiB/s wr, 8 op/s 2024-04-06T14:21:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-06T14:21:35.538 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: Cluster is now healthy 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6837/442878082,v1:172.21.15.134:6838/442878082] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6836/745783224,v1:172.21.15.134:6839/745783224] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.134:6834/3757701937,v1:172.21.15.134:6835/3757701937] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6834/1460644165,v1:172.21.15.105:6835/1460644165] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6836/151761867,v1:172.21.15.105:6837/151761867] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: mds.? [v2:172.21.15.105:6838/2977723885,v1:172.21.15.105:6839/2977723885] up:boot 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1462829521' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: fsmap 12 up:standby 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2024-04-06T14:21:35.539 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2024-04-06T14:21:35.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='mgr.15288 172.21.15.90:0/2985539916' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-04-06T14:21:35.540 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:35 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:35.656 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.655+0000 7f62677fe700 1 -- 172.21.15.90:0/1236862169 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v106) v1 ==== 181+0+0 (secure 0 0 0) 0x7f62680995c0 con 0x7f6270105a70 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 msgr2=0x7f625c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 0x7f625c0a3750 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f626c00b5c0 tx=0x7f626c005fb0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 msgr2=0x7f627019b790 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 secure :-1 s=READY pgs=945 cs=0 l=1 rev1=1 crypto rx=0x7f6268007ee0 tx=0x7f626800d3b0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 shutdown_connections 2024-04-06T14:21:35.659 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7f625c0a1290 0x7f625c0a3750 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7f6270105a70 0x7f627019b790 unknown :-1 s=CLOSED pgs=945 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62700ffa60 0x7f6270196eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 --2- 172.21.15.90:0/1236862169 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7f6270100420 0x7f62701973f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 >> 172.21.15.90:0/1236862169 conn(0x7f62700fb570 msgr2=0x7f6270069130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 shutdown_connections 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.658+0000 7f627719e700 1 -- 172.21.15.90:0/1236862169 wait complete. 2024-04-06T14:21:35.660 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'cephfs_metadata' removed 2024-04-06T14:21:35.673 DEBUG:teuthology.orchestra.run.smithi090:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-04-06T14:21:35.804 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- 172.21.15.90:0/2349159747 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 msgr2=0x7fb504100250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 --2- 172.21.15.90:0/2349159747 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504100250 secure :-1 s=READY pgs=946 cs=0 l=1 rev1=1 crypto rx=0x7fb4f8009990 tx=0x7fb4f8009ca0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- 172.21.15.90:0/2349159747 shutdown_connections 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 --2- 172.21.15.90:0/2349159747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504104f90 0x7fb50410cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 --2- 172.21.15.90:0/2349159747 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504100250 unknown :-1 s=CLOSED pgs=946 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 --2- 172.21.15.90:0/2349159747 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504100820 0x7fb504104860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- 172.21.15.90:0/2349159747 >> 172.21.15.90:0/2349159747 conn(0x7fb504076d80 msgr2=0x7fb504077190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- 172.21.15.90:0/2349159747 shutdown_connections 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- 172.21.15.90:0/2349159747 wait complete. 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 Processor -- start 2024-04-06T14:21:35.805 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.804+0000 7fb509aa1700 1 -- start start 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 --2- >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504104f90 0x7fb504101e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 0x7fb5041a5420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 -- --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5041a59f0 con 0x7fb5040ffe70 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5041a5b60 con 0x7fb504102380 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 -- --> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5041a5d20 con 0x7fb504104f90 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 --2- >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.90:3300/0 says I am v2:172.21.15.90:60532/0 (socket says 172.21.15.90:60532) 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 -- 172.21.15.90:0/4122502290 learned_addr learned my addr 172.21.15.90:0/4122502290 (peer_addr_for_me v2:172.21.15.90:0/0) 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb503fff700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 0x7fb5041a5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb502ffd700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504104f90 0x7fb504101e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:35.808 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 -- 172.21.15.90:0/4122502290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504104f90 msgr2=0x7fb504101e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504104f90 0x7fb504101e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 -- 172.21.15.90:0/4122502290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 msgr2=0x7fb5041a5420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 0x7fb5041a5420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 -- 172.21.15.90:0/4122502290 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4f80095f0 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb503fff700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 0x7fb5041a5420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb5037fe700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7fb4f8005770 tx=0x7fb4f8009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4f801d070 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb4f800fc70 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5041a5fa0 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4f8022ce0 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.805+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5041a6360 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.806+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb504109210 con 0x7fb5040ffe70 2024-04-06T14:21:35.809 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.807+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 4 ==== mgrmap(e 25) v1 ==== 136004+0+0 (secure 0 0 0) 0x7fb4f8022470 con 0x7fb5040ffe70 2024-04-06T14:21:35.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.808+0000 7fb500ff9700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 0x7fb4ec0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-06T14:21:35.810 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.808+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 5 ==== osd_map(106..106 src has 1..106) v4 ==== 8245+0+0 (secure 0 0 0) 0x7fb4f8022e40 con 0x7fb5040ffe70 2024-04-06T14:21:35.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.808+0000 7fb502ffd700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 0x7fb4ec0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-06T14:21:35.811 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.808+0000 7fb502ffd700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 0x7fb4ec0a3750 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7fb4f4005fd0 tx=0x7fb4f4005f60 comp rx=0 tx=0).ready entity=mgr.15288 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-06T14:21:35.813 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:35.812+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+182936 (secure 0 0 0) 0x7fb4f80976d0 con 0x7fb5040ffe70 2024-04-06T14:21:36.010 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.010+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 --> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7fb504102de0 con 0x7fb5040ffe70 2024-04-06T14:21:36.668 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.668+0000 7fb500ff9700 1 -- 172.21.15.90:0/4122502290 <== mon.0 v2:172.21.15.90:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v107) v1 ==== 169+0+0 (secure 0 0 0) 0x7fb4f8027020 con 0x7fb5040ffe70 2024-04-06T14:21:36.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.672+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 msgr2=0x7fb4ec0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:36.672 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.672+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 0x7fb4ec0a3750 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7fb4f4005fd0 tx=0x7fb4f4005f60 comp rx=0 tx=0).stop 2024-04-06T14:21:36.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.672+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 msgr2=0x7fb504101900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-06T14:21:36.673 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.672+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 secure :-1 s=READY pgs=947 cs=0 l=1 rev1=1 crypto rx=0x7fb4f8005770 tx=0x7fb4f8009580 comp rx=0 tx=0).stop 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 shutdown_connections 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:6800/2273339784,v1:172.21.15.90:6801/2273339784] conn(0x7fb4ec0a1290 0x7fb4ec0a3750 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb504102380 0x7fb5041a5420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.90:3300/0,v1:172.21.15.90:6789/0] conn(0x7fb5040ffe70 0x7fb504101900 unknown :-1 s=CLOSED pgs=947 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 --2- 172.21.15.90:0/4122502290 >> [v2:172.21.15.134:3300/0,v1:172.21.15.134:6789/0] conn(0x7fb504104f90 0x7fb504101e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-06T14:21:36.676 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.676+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 >> 172.21.15.90:0/4122502290 conn(0x7fb504076d80 msgr2=0x7fb504069640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-06T14:21:36.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.677+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 shutdown_connections 2024-04-06T14:21:36.677 INFO:teuthology.orchestra.run.smithi090.stderr:2024-04-06T14:21:36.677+0000 7fb509aa1700 1 -- 172.21.15.90:0/4122502290 wait complete. 2024-04-06T14:21:36.678 INFO:teuthology.orchestra.run.smithi090.stderr:pool 'cephfs_data' removed 2024-04-06T14:21:36.698 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:21:36.823 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:21:36.863 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:21:36.911 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-06T14:21:36.912 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:36 smithi134 ceph-mon[47549]: osdmap e106: 12 total, 12 up, 12 in 2024-04-06T14:21:36.912 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:36 smithi134 ceph-mon[47549]: from='client.? 172.21.15.90:0/4122502290' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:36.912 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:36 smithi134 ceph-mon[47549]: pgmap v511: 65 pgs: 65 active+clean; 5.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:21:36.917 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-06T14:21:36.917 DEBUG:teuthology.orchestra.run.smithi090:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-06T14:21:36.957 DEBUG:teuthology.orchestra.run.smithi105:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-06T14:21:36.977 DEBUG:teuthology.orchestra.run.smithi134:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-06T14:21:36.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-06T14:21:36.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:36 smithi105 ceph-mon[46969]: osdmap e106: 12 total, 12 up, 12 in 2024-04-06T14:21:36.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:36 smithi105 ceph-mon[46969]: from='client.? 172.21.15.90:0/4122502290' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:36.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:36 smithi105 ceph-mon[46969]: pgmap v511: 65 pgs: 65 active+clean; 5.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:21:37.004 INFO:tasks.cephadm:Stopping all daemons... 2024-04-06T14:21:37.004 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-06T14:21:37.004 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a 2024-04-06T14:21:37.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/1236862169' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-04-06T14:21:37.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:36 smithi090 ceph-mon[43420]: osdmap e106: 12 total, 12 up, 12 in 2024-04-06T14:21:37.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:36 smithi090 ceph-mon[43420]: from='client.? 172.21.15.90:0/4122502290' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-06T14:21:37.041 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:36 smithi090 ceph-mon[43420]: pgmap v511: 65 pgs: 65 active+clean; 5.1 GiB data, 21 GiB used, 1.0 TiB / 1.0 TiB avail 2024-04-06T14:21:37.474 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:37 smithi090 systemd[1]: Stopping Ceph mon.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:37.474 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a[43397]: 2024-04-06T14:21:37.466+0000 7f919abae700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:37.474 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a[43397]: 2024-04-06T14:21:37.466+0000 7f919abae700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-06T14:21:38.289 INFO:journalctl@ceph.mon.a.smithi090.stdout:Apr 06 14:21:37 smithi090 bash[128735]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-a 2024-04-06T14:21:38.431 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.a.service' 2024-04-06T14:21:38.551 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:38.552 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-06T14:21:38.552 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2024-04-06T14:21:38.552 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.b 2024-04-06T14:21:38.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:38 smithi105 systemd[1]: Stopping Ceph mon.b for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:39.996 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-b[46946]: 2024-04-06T14:21:39.553+0000 7f19fce87700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:39.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-b[46946]: 2024-04-06T14:21:39.553+0000 7f19fce87700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-06T14:21:40.645 INFO:journalctl@ceph.mon.b.smithi105.stdout:Apr 06 14:21:40 smithi105 bash[90763]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-b 2024-04-06T14:21:41.951 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.b.service' 2024-04-06T14:21:42.203 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:42.203 INFO:tasks.cephadm.mon.c:Stopped mon.b 2024-04-06T14:21:42.203 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2024-04-06T14:21:42.203 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.c 2024-04-06T14:21:42.539 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:42 smithi134 systemd[1]: Stopping Ceph mon.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:42.828 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-c[47525]: 2024-04-06T14:21:42.539+0000 7fe7f22bd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:42.829 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-c[47525]: 2024-04-06T14:21:42.539+0000 7fe7f22bd700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-06T14:21:43.142 INFO:journalctl@ceph.mon.c.smithi134.stdout:Apr 06 14:21:42 smithi134 bash[92120]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mon-c 2024-04-06T14:21:43.232 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mon.c.service' 2024-04-06T14:21:43.288 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:43.288 INFO:tasks.cephadm.mon.c:Stopped mon.c 2024-04-06T14:21:43.288 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-06T14:21:43.288 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x 2024-04-06T14:21:43.571 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:21:43 smithi090 systemd[1]: Stopping Ceph mgr.x for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:43.571 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:21:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x[43673]: 2024-04-06T14:21:43.569+0000 7f9226cf4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:44.038 INFO:journalctl@ceph.mgr.x.smithi090.stdout:Apr 06 14:21:43 smithi090 bash[128861]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-x 2024-04-06T14:21:44.062 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.x.service' 2024-04-06T14:21:44.099 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:44.099 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-06T14:21:44.099 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-06T14:21:44.099 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.y 2024-04-06T14:21:44.448 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:21:44 smithi105 systemd[1]: Stopping Ceph mgr.y for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:44.746 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:21:44 smithi105 bash[90903]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-y 2024-04-06T14:21:44.747 INFO:journalctl@ceph.mgr.y.smithi105.stdout:Apr 06 14:21:44 smithi105 systemd[1]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-06T14:21:44.773 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.y.service' 2024-04-06T14:21:44.811 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:44.811 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-06T14:21:44.811 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2024-04-06T14:21:44.812 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.z 2024-04-06T14:21:45.140 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:21:44 smithi134 systemd[1]: Stopping Ceph mgr.z for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:45.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:21:45 smithi134 bash[92262]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mgr-z 2024-04-06T14:21:45.393 INFO:journalctl@ceph.mgr.z.smithi134.stdout:Apr 06 14:21:45 smithi134 systemd[1]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.z.service: Main process exited, code=exited, status=143/n/a 2024-04-06T14:21:45.496 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mgr.z.service' 2024-04-06T14:21:45.533 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:45.533 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2024-04-06T14:21:45.533 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-06T14:21:45.533 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.0 2024-04-06T14:21:46.038 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:21:45 smithi090 systemd[1]: Stopping Ceph osd.0 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:46.038 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:21:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:21:45.770+0000 7fba6e126700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:46.039 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:21:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:21:45.770+0000 7fba6e126700 -1 osd.0 107 *** Got signal Terminated *** 2024-04-06T14:21:46.039 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:21:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0[62653]: 2024-04-06T14:21:45.770+0000 7fba6e126700 -1 osd.0 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:21:52.038 INFO:journalctl@ceph.osd.0.smithi090.stdout:Apr 06 14:21:51 smithi090 bash[129009]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-0 2024-04-06T14:21:53.000 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.0.service' 2024-04-06T14:21:53.041 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:21:53.041 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-06T14:21:53.042 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-06T14:21:53.042 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.1 2024-04-06T14:21:53.496 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:21:53 smithi105 systemd[1]: Stopping Ceph osd.1 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:21:53.497 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:21:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:21:53.274+0000 7fb34b684700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:21:53.497 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:21:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:21:53.274+0000 7fb34b684700 -1 osd.1 107 *** Got signal Terminated *** 2024-04-06T14:21:53.497 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:21:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1[51974]: 2024-04-06T14:21:53.274+0000 7fb34b684700 -1 osd.1 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:21:59.247 INFO:journalctl@ceph.osd.1.smithi105.stdout:Apr 06 14:21:59 smithi105 bash[91050]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-1 2024-04-06T14:22:00.186 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.1.service' 2024-04-06T14:22:00.227 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:00.228 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-06T14:22:00.228 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-06T14:22:00.228 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.2 2024-04-06T14:22:00.643 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:22:00 smithi134 systemd[1]: Stopping Ceph osd.2 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:00.643 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:22:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:22:00.464+0000 7fd9eae4f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:00.643 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:22:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:22:00.464+0000 7fd9eae4f700 -1 osd.2 107 *** Got signal Terminated *** 2024-04-06T14:22:00.643 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:22:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2[53342]: 2024-04-06T14:22:00.464+0000 7fd9eae4f700 -1 osd.2 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:06.643 INFO:journalctl@ceph.osd.2.smithi134.stdout:Apr 06 14:22:06 smithi134 bash[92415]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-2 2024-04-06T14:22:07.511 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.2.service' 2024-04-06T14:22:07.552 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:07.552 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-06T14:22:07.552 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-06T14:22:07.552 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.3 2024-04-06T14:22:08.038 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:07 smithi090 systemd[1]: Stopping Ceph osd.3 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:08.538 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:08 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:22:08.057+0000 7f565bac7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:08.538 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:08 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:22:08.057+0000 7f565bac7700 -1 osd.3 107 *** Got signal Terminated *** 2024-04-06T14:22:08.538 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:08 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:22:08.057+0000 7f565bac7700 -1 osd.3 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:12.788 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:12 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3[70538]: 2024-04-06T14:22:12.404+0000 7f56540b5700 -1 osd.3 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.775099+0000 front 2024-04-06T14:21:47.775036+0000 (oldest deadline 2024-04-06T14:22:11.875216+0000) 2024-04-06T14:22:13.496 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:13 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:13.052+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:14.002 INFO:journalctl@ceph.osd.3.smithi090.stdout:Apr 06 14:22:13 smithi090 bash[129267]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-3 2024-04-06T14:22:14.003 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:13 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:13.680+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:14.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:14 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:14.002+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:14.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:14 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:14.053+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:14.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:14 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:14.655+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:14.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:14 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:14.506+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:14.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:14 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:14.467+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:15.241 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:14 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:14.986+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:15.249 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.3.service' 2024-04-06T14:22:15.290 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:15.290 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-06T14:22:15.290 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-06T14:22:15.291 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.4 2024-04-06T14:22:15.320 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:15 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:15.069+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:15.747 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:15 smithi105 systemd[1]: Stopping Ceph osd.4 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:15.747 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:15 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:22:15.530+0000 7f3f7cfda700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:15.747 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:15 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:22:15.530+0000 7f3f7cfda700 -1 osd.4 107 *** Got signal Terminated *** 2024-04-06T14:22:15.747 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:15 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:22:15.530+0000 7f3f7cfda700 -1 osd.4 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:15.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:15 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:15.476+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:15.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:15 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:15.468+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:15.995 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:15 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:15.607+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:16.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:15 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:15.995+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:16.496 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:16 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:16.036+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:16.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:16 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:16.438+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:16.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:16 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:16.489+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:16.973 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:16.596+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:17.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:16 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:16.974+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:17.496 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:17 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:17.031+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:17.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:17.403+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:17.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:17 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:17.531+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:17.952 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:17.603+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:18.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:17 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:17.952+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:18.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:18 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:18.025+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:18.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:18.425+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:18.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:18 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:18.506+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:18.980 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:18.553+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:19.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:18.980+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:19.496 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:19.001+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:19.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:19.445+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:19.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:19 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:19.500+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:20.029 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:19 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:19.559+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:20.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:19 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:19.982+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:20.247 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:20 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4[60654]: 2024-04-06T14:22:20.029+0000 7f3f795d0700 -1 osd.4 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.499354+0000 front 2024-04-06T14:21:54.499383+0000 (oldest deadline 2024-04-06T14:22:19.799467+0000) 2024-04-06T14:22:20.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:20.030+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:20.875 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:20.456+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:20.876 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:20.422+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:20.876 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:20.422+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:21.036 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:20 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:20.572+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:21.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:20 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:20.875+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:21.286 INFO:journalctl@ceph.osd.4.smithi105.stdout:Apr 06 14:22:21 smithi105 bash[91312]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-4 2024-04-06T14:22:21.286 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:21.011+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:21.286 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:21.011+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:21.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:21.036+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:21.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:21.036+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:21.604 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:21 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:21.285+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:21.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:21.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:21.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:21.423+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:21.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:21.423+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:22.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:21 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:21.615+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:22.312 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:22 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:22.052+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:22.312 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:22 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:22.052+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:22.342 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.4.service' 2024-04-06T14:22:22.392 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:21 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:21.918+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:22.422 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:22.423 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-06T14:22:22.423 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-06T14:22:22.423 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.5 2024-04-06T14:22:22.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:22.049+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:22.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:22.049+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:22.678 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:22.440+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:22.423+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:22.423+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 systemd[1]: Stopping Ceph osd.5 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:22.678+0000 7f1f7ae4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:22.678+0000 7f1f7ae4e700 -1 osd.5 107 *** Got signal Terminated *** 2024-04-06T14:22:22.679 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:22.678+0000 7f1f7ae4e700 -1 osd.5 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:22.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:22 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:22.311+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:23.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:22.649+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:23.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:22.649+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:23.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:22 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:22.911+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:23.331 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:23.064+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:23.332 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:23.064+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:23.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:23.052+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:23.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:23.052+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:23.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:23 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:23.331+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:23.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:23.422+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:23.893 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:23.422+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:23.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:23.461+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:24.003 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:23.674+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:24.003 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:23 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:23.674+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:24.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:24.003+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:24.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:24.003+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:24.379 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:23 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:23.897+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:24.496 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:24 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:24.314+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:24.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:24 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:24.077+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:24.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:24 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:24.077+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:24.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:24.379+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:24.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:24.379+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:24.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:24.424+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:25.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:24.641+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:25.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:24.641+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:25.142 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:24 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:24.850+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:25.497 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:25 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:25.300+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:25.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:25 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:25.058+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:25.498 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:25 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:25.058+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:25.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:25.045+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:25.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:25.045+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:25.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:25 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:25.336+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:25.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:25 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:25.336+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:25.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:25 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:25.431+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:26.017 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:25.595+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:26.017 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:25 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:25.595+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:26.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:25 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:25.879+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:26.288 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:26 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:26.016+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:26.289 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:26 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:26.016+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:26.326 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:26 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:26.057+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:26.326 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:26 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:26.057+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:26.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:26 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:26.302+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:26.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:26 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:26.302+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:26.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:26 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:26.390+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:26.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:26 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:26.325+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:27.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:26 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:26.625+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:27.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:26 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:26.625+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:27.270 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:26 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:26.908+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:27.358 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:27 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:27.066+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:27.358 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:27 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:27.066+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:27.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:27 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:27.047+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:27.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:27 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:27.047+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:27.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:27 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:27.351+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:27.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:27 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:27.270+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:50.151418+0000 front 2024-04-06T14:21:50.151228+0000 (oldest deadline 2024-04-06T14:22:14.251368+0000) 2024-04-06T14:22:27.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:27 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:27.270+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:57.152515+0000 front 2024-04-06T14:21:57.152546+0000 (oldest deadline 2024-04-06T14:22:19.452556+0000) 2024-04-06T14:22:27.643 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:27 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5[61796]: 2024-04-06T14:22:27.270+0000 7f1f7343c700 -1 osd.5 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.753307+0000 front 2024-04-06T14:22:04.753357+0000 (oldest deadline 2024-04-06T14:22:27.053431+0000) 2024-04-06T14:22:27.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:27 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:27.358+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:27.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:27 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:27.358+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:28.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:27 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:27.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:28.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:27 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:27.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:28.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:27 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:27.871+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:28.361 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:28.040+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:28.362 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:28.040+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:28.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:28 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:28.089+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:28.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:28 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:28.089+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:28.646 INFO:journalctl@ceph.osd.5.smithi134.stdout:Apr 06 14:22:28 smithi134 bash[92671]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-5 2024-04-06T14:22:28.646 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:28 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:28.307+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:28.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:28.361+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:28.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:28.361+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:29.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:28 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:28.601+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:29.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:28 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:28.601+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:29.292 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:28 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:28.897+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:29.292 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:28 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:28.897+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:29.409 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:29.016+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:29.409 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:29.016+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:29.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:29 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:29.084+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:29.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:29 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:29.084+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:29.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:29 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:29.084+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:29.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:29 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:29.292+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:29.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:29.409+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:29.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:29.409+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:29.774 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.5.service' 2024-04-06T14:22:29.815 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:29.816 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-06T14:22:29.816 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-06T14:22:29.816 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.6 2024-04-06T14:22:30.038 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:29 smithi090 systemd[1]: Stopping Ceph osd.6 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:30.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:29 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:29.635+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:30.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:29 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:29.635+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:30.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:29 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:29.943+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:30.143 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:29 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:29.943+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:30.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:29.970+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:30.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:29 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:29.970+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.067+0000 7fb4503c4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.067+0000 7fb4503c4700 -1 osd.6 107 *** Got signal Terminated *** 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.067+0000 7fb4503c4700 -1 osd.6 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.092+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.092+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:30.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:30.092+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:30.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:30 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:30.331+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:30.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:30 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:30.331+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:30.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:30 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:30.438+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:30.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:30 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:30.438+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:31.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:30.673+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:31.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:30.673+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:31.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:30 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:30.673+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:31.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:30 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:30.923+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:31.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:30 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:30.923+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:31.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:30 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:30.923+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:31.380 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:30 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:30.971+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:31.380 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:30 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:30.971+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:31.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:31.101+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:31.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:31.101+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:31.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:31.101+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:31.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:31 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:31.380+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:31.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:31 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:31.380+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:31.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:31.430+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:31.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:31.430+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:32.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:31.663+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:32.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:31.663+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:32.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:31 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:31.663+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:32.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:31.897+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:32.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:31.897+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:32.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:31 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:31.897+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:32.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:31 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:31.989+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:32.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:31 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:31.989+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:32.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:32.110+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:32.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:32.110+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:32.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:32.110+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:32.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:32 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:32.404+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:32.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:32 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:32.404+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:32.892 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:32.408+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:32.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:32.408+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:33.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:32.687+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:33.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:32.687+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:33.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:32 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:32.687+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:33.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:32 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:32.849+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:33.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:32 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:32.849+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:33.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:32 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:32.849+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:33.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:32.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:33.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:32 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:32.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:33.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:33.151+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:33.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:33.151+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:33.539 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:33.151+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:33.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:33.455+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:33.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:33.455+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:33.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:33.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:33.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:33.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:34.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:33.672+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:34.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:33.672+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:34.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:33 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:33.672+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:34.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:33.818+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:34.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:33.818+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:34.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:33.818+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:34.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:33.957+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:34.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:33 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:33.957+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:34.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:34.187+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.172756+0000 front 2024-04-06T14:21:47.172810+0000 (oldest deadline 2024-04-06T14:22:13.073022+0000) 2024-04-06T14:22:34.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:34.187+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373882+0000 front 2024-04-06T14:21:55.373861+0000 (oldest deadline 2024-04-06T14:22:20.673990+0000) 2024-04-06T14:22:34.538 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6[78520]: 2024-04-06T14:22:34.187+0000 7fb4489b2700 -1 osd.6 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:03.574997+0000 front 2024-04-06T14:22:03.575033+0000 (oldest deadline 2024-04-06T14:22:28.275016+0000) 2024-04-06T14:22:34.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:34 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:34.463+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:34.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:34 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:34.463+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:34.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:34 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:34.463+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:34.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:34 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:34.463+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:35.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:34.635+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:35.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:34.635+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:35.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:34 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:34.635+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:35.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:34 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:34.844+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:35.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:34 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:34.844+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:35.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:34 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:34.844+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:35.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:35 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:35.007+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:35.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:35 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:35.007+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:35.829 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:35 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:35.511+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:35.829 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:35 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:35.511+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:35.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:35 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:35.437+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:35.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:35 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:35.437+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:35.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:35 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:35.437+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:36.038 INFO:journalctl@ceph.osd.6.smithi090.stdout:Apr 06 14:22:35 smithi090 bash[129520]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-6 2024-04-06T14:22:36.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:35 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:35.664+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:36.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:35 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:35.664+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:36.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:35 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:35.664+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:36.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:35 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:35.829+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:36.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:35 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:35.829+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:36.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:35 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:35.829+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:36.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:36.037+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:36.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:36.037+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:36.659 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:36.658+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:36.837 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:36.523+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:36.837 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:36.523+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:36.837 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:36.523+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:36.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:36.402+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:36.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:36.402+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:36.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:36 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:36.402+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:37.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:36.658+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:37.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:36.658+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:37.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:36 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:36.658+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:37.089 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.6.service' 2024-04-06T14:22:37.129 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:37.129 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-06T14:22:37.130 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-06T14:22:37.130 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.7 2024-04-06T14:22:37.161 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:36.837+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:37.161 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:36.837+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:37.161 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:36.837+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:37.161 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:36 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:36.837+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:37.363 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:37 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:37.016+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:37.364 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:37 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:37.016+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:37.497 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 systemd[1]: Stopping Ceph osd.7 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:37.497 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.360+0000 7fbbe3d40700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:37.497 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.361+0000 7fbbe3d40700 -1 osd.7 107 *** Got signal Terminated *** 2024-04-06T14:22:37.497 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.361+0000 7fbbe3d40700 -1 osd.7 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:37.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:37 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:37.363+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:37.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:37 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:37.363+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:37.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:37 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:37.363+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:37.871 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.545+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:37.871 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.545+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:37.871 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:37.545+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:38.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:37.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:38.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:37.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:38.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:37.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:38.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:37 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:37.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:38.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:37.871+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:38.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:37.871+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:38.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:37.871+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:38.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:37 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:37.871+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:38.323 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:38.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:38.323 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:38.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:38.323 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:38.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:38.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:38.322+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:38.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:38.322+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:38.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:38 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:38.322+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:38.827 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:38.499+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:38.827 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:38.499+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:38.827 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:38.499+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:39.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:38.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:39.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:38.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:39.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:38.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:39.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:38 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:38.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:39.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:38.827+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:39.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:38.827+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:39.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:38.827+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:39.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:38.827+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:39.282 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:39.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:39.282 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:39.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:39.282 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:39.001+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:39.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:39.281+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:39.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:39.281+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:39.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:39 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:39.281+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:39.792 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:39.505+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:39.792 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:39.505+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:39.792 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:39.505+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:40.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:39.634+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:40.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:39.634+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:40.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:39.634+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:40.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:39 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:39.634+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:40.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:39.792+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:40.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:39.792+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:40.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:39.792+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:40.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:39 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:39.792+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:40.301 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:40.021+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:40.301 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:40.021+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:40.301 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:40.021+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:40.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:40.301+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:40.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:40.301+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:40.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:40 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:40.301+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:40.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:40.473+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:40.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:40.473+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:40.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:40.473+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:41.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:40.665+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:41.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:40.665+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:41.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:40.665+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:41.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:40 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:40.665+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:41.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:40.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:41.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:40.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:41.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:40.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:41.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:40 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:40.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:41.310 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.009+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:41.311 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.009+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:41.311 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.009+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:41.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:41.310+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:41.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:41.310+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:41.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:41.310+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:41.746 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:41.429+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:55.373233+0000 front 2024-04-06T14:21:55.373415+0000 (oldest deadline 2024-04-06T14:22:20.673354+0000) 2024-04-06T14:22:41.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:41.429+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:00.673858+0000 front 2024-04-06T14:22:00.673922+0000 (oldest deadline 2024-04-06T14:22:26.573831+0000) 2024-04-06T14:22:41.747 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7[69383]: 2024-04-06T14:22:41.429+0000 7fbbdc32e700 -1 osd.7 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.875131+0000 front 2024-04-06T14:22:11.875207+0000 (oldest deadline 2024-04-06T14:22:35.975035+0000) 2024-04-06T14:22:42.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:41.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:42.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:41.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:42.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:41.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:42.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:41.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:42.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:41 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:41.622+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:42.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:41.843+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:42.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:41.843+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:42.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:41.843+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:42.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:41 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:41.843+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:42.292 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.992+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:42.292 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.992+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:42.292 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:41 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:41.992+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:42.642 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:42.292+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:42.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:42.292+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:42.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:42.292+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:42.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:42.292+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:43.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:42.670+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:43.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:42.670+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:43.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:42.670+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:43.040 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:42.670+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:43.040 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:42 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:42.670+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:43.181 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:42.794+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:43.182 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:42.794+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:43.182 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:42.794+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:43.182 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:42.794+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:43.295 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:42.968+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:43.295 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:42.968+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:43.295 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:42.968+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:43.295 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:42 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:42.968+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:43.496 INFO:journalctl@ceph.osd.7.smithi105.stdout:Apr 06 14:22:43 smithi105 bash[91566]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-7 2024-04-06T14:22:43.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:43.294+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:43.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:43.295+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:43.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:43.295+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:43.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:43.295+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:43.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:43 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:43.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:43.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:43 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:43.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:43.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:43 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:43.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:43.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:43 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:43.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:44.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:43.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:44.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:43.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:44.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:43.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:44.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:43.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:44.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:43 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:43.661+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:44.339 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:43.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:44.339 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:43.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:44.339 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:43.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:44.339 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:43 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:43.986+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:44.380 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.7.service' 2024-04-06T14:22:44.461 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:44.461 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-06T14:22:44.461 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2024-04-06T14:22:44.461 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.8 2024-04-06T14:22:44.643 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 systemd[1]: Stopping Ceph osd.8 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:44.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:44.338+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:44.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:44.338+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:44.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:44.338+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:44.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:44.338+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:44.978 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.691+0000 7f7c58070700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:44.978 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.691+0000 7f7c58070700 -1 osd.8 107 *** Got signal Terminated *** 2024-04-06T14:22:44.978 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.691+0000 7f7c58070700 -1 osd.8 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:44.978 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.976+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:44.978 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.976+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:45.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:44 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:44.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:45.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:44 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:44.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:45.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:44 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:44.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:45.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:44 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:44.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:45.039 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:44 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:44.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:45.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:44 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:44.780+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:45.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:44 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:44.780+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:45.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:44 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:44.780+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:45.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:44 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:44.780+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:45.350 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.977+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:45.351 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:44 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:44.977+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:45.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:45 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:45.351+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:45.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:45 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:45.351+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:45.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:45 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:45.351+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:45.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:45 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:45.351+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:46.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:45.653+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:46.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:45.653+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:46.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:45.653+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:46.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:45.653+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:46.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:45 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:45.653+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:46.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:45 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:45.784+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:46.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:45 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:45.784+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:46.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:45 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:45.784+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:46.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:45 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:45.784+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:46.318 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:46.013+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:46.318 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:46.013+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:46.318 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:46.013+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:46.319 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:46.013+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:46.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:46.318+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:46.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:46.318+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:46.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:46.318+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:46.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:46.318+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:47.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:46.666+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:47.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:46.666+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:47.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:46.666+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:47.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:46.666+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:47.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:46 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:46.666+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:47.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:46 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:46.756+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:47.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:46 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:46.756+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:47.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:46 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:46.756+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:47.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:46 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:46.756+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:47.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:46 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:46.756+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:47.346 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:47.036+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:47.346 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:47.036+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:47.346 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:47.036+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:47.346 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:47.036+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:47.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:47.345+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:47.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:47.345+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:47.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:47.345+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:47.643 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:47 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:47.345+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:47.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:47 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:47.742+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:47.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:47 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:47.742+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:47.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:47 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:47.742+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:47.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:47 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:47.742+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:47.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:47 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:47.742+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:48.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:47 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:47.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:48.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:47 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:47.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:48.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:47 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:47.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:48.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:47 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:47.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:48.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:47 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:47.659+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:48.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:48.017+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:48.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:48.017+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:48.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:48.017+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:48.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:48.017+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:48.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:48.392+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:48.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:48.392+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:48.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:48.392+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:48.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:48 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:48.392+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:49.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:48.690+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:49.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:48.690+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:49.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:48.690+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:49.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:48.690+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:49.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:48 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:48.690+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:49.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:48 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:48.765+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:49.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:49.018+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.981243+0000 front 2024-04-06T14:21:54.981137+0000 (oldest deadline 2024-04-06T14:22:20.281160+0000) 2024-04-06T14:22:49.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:49.018+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:02.482716+0000 front 2024-04-06T14:22:02.482661+0000 (oldest deadline 2024-04-06T14:22:28.382900+0000) 2024-04-06T14:22:49.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:49.018+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:11.883943+0000 front 2024-04-06T14:22:11.883965+0000 (oldest deadline 2024-04-06T14:22:37.784055+0000) 2024-04-06T14:22:49.393 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8[69757]: 2024-04-06T14:22:49.018+0000 7f7c54666700 -1 osd.8 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:18.285069+0000 front 2024-04-06T14:22:18.285100+0000 (oldest deadline 2024-04-06T14:22:42.385131+0000) 2024-04-06T14:22:49.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:49.424+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:49.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:49.424+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:49.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:49.424+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:49.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:49 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:49.424+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:49.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:49.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:49.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:49.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:49.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:49.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:49 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:49.726+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:50.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:49.646+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:50.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:49.646+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:50.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:49.646+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:50.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:49.646+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:50.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:49 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:49.646+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:50.893 INFO:journalctl@ceph.osd.8.smithi134.stdout:Apr 06 14:22:50 smithi134 bash[92926]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-8 2024-04-06T14:22:50.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:50.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:50.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:50.452+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:50.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:50.452+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:50.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:50.452+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:51.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:50 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:50.639+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:51.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:50 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:50.639+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:51.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:50 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:50.639+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:51.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:50 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:50.639+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:51.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:50 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:50.639+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:51.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:50 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:50.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:51.763 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:51 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:51.441+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:51.763 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:51 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:51.441+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:51.763 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:51 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:51.441+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:51.763 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:51 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:51.441+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:51.770 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.8.service' 2024-04-06T14:22:51.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:51.810 INFO:tasks.cephadm.osd.8:Stopped osd.8 2024-04-06T14:22:51.810 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2024-04-06T14:22:51.810 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.9 2024-04-06T14:22:51.856 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:51.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:51.856 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:51.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:51.856 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:51.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:51.856 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:51.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:51.856 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:51.593+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:52.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:52.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:52.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:52.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:52.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:52.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:51 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:51.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:52.288 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:51 smithi090 systemd[1]: Stopping Ceph osd.9 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:52.289 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.066+0000 7f2acacb8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:52.289 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.066+0000 7f2acacb8700 -1 osd.9 107 *** Got signal Terminated *** 2024-04-06T14:22:52.289 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.066+0000 7f2acacb8700 -1 osd.9 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:52.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:52 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:52.420+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:52.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:52 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:52.420+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:52.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:52 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:52.420+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:52.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:52 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:52.420+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:52.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:52 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:52.420+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:53.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.580+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:53.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.580+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:53.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.580+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:53.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.580+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:53.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:52 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:52.580+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:53.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:53.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:53.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:53.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:53.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:53.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:52 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:52.798+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:53.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:53 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:53.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:53.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:53 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:53.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:53.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:53 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:53.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:53.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:53 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:53.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:53.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:53 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:53.451+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:54.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:53.571+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:54.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:53.571+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:54.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:53.571+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:54.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:53.571+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:54.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:53 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:53.571+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:54.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:54.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:54.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:54.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:54.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:54.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:53 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:53.752+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:54.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:54.499+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:54.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:54.499+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:54.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:54.499+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:54.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:54.499+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:54.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:54.499+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:55.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:54 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:54.552+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:55.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:54 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:54.552+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:55.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:54 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:54.552+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:55.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:54 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:54.552+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:55.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:54 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:54.552+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:55.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:55.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:55.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:55.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:55.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:55.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:54 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:54.783+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:55.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:55 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:55.506+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:55.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:55 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:55.506+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:55.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:55 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:55.506+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:55.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:55 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:55.506+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:55.788 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:55 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:55.506+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:55.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:55.516+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:55.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:55.516+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:55.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:55.516+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:55.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:55.516+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:55.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:55 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:55.516+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:56.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:55 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:55.821+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:56.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:56.532+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:56.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:56.532+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:56.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:56.532+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:56.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:56.532+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:56.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:56 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:56.532+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:57.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:56 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:56.541+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:47.643554+0000 front 2024-04-06T14:21:47.643526+0000 (oldest deadline 2024-04-06T14:22:12.943572+0000) 2024-04-06T14:22:57.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:56 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:56.541+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:56.444683+0000 front 2024-04-06T14:21:56.444606+0000 (oldest deadline 2024-04-06T14:22:21.744629+0000) 2024-04-06T14:22:57.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:56 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:56.541+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.645968+0000 front 2024-04-06T14:22:04.645866+0000 (oldest deadline 2024-04-06T14:22:30.546007+0000) 2024-04-06T14:22:57.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:56 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:56.541+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.546486+0000 front 2024-04-06T14:22:10.546517+0000 (oldest deadline 2024-04-06T14:22:36.446592+0000) 2024-04-06T14:22:57.038 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:56 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9[87116]: 2024-04-06T14:22:56.541+0000 7f2ac72ae700 -1 osd.9 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:16.447116+0000 front 2024-04-06T14:22:16.447131+0000 (oldest deadline 2024-04-06T14:22:41.147113+0000) 2024-04-06T14:22:57.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:57.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:56 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:56.790+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:57.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:57 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:57.503+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:22:58.246 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:58.247 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:57 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:57.754+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:22:58.288 INFO:journalctl@ceph.osd.9.smithi090.stdout:Apr 06 14:22:57 smithi090 bash[129781]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-9 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:58.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:58.485+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:22:59.062 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.9.service' 2024-04-06T14:22:59.103 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:22:59.103 INFO:tasks.cephadm.osd.9:Stopped osd.9 2024-04-06T14:22:59.103 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2024-04-06T14:22:59.103 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.10 2024-04-06T14:22:59.134 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.768+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:59.134 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:59.134 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:59.135 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:59.135 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:59.135 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:59.135 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:58 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:58.769+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:22:59.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 systemd[1]: Stopping Ceph osd.10 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:22:59.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.333+0000 7f1ed854e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:22:59.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.333+0000 7f1ed854e700 -1 osd.10 107 *** Got signal Terminated *** 2024-04-06T14:22:59.497 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.333+0000 7f1ed854e700 -1 osd.10 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:22:59.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:22:59 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:22:59.511+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:22:59.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:22:59.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:22:59 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:22:59.722+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:00.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:00 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:00.462+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:00.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:23:00.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:23:00.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:23:00.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:23:00.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:23:00.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:23:00.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:00 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:00.719+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:01.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:01 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:01.512+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:01.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:23:01.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:01 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:01.729+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:23:02.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:02.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:02.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:02.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:02.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:02.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:02 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:02.547+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:02.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:23:02.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:23:02.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:23:02.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:23:02.998 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:23:02.998 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:23:02.998 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:02 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:02.706+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:03.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:03 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:03.576+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:03.996 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.733502+0000 front 2024-04-06T14:21:48.733484+0000 (oldest deadline 2024-04-06T14:22:12.833497+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6804 osd.1 since back 2024-04-06T14:21:54.534605+0000 front 2024-04-06T14:21:54.534630+0000 (oldest deadline 2024-04-06T14:22:20.434772+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:05.036382+0000 front 2024-04-06T14:22:05.036211+0000 (oldest deadline 2024-04-06T14:22:30.336118+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:10.837030+0000 front 2024-04-06T14:22:10.837146+0000 (oldest deadline 2024-04-06T14:22:36.137035+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.637796+0000 front 2024-04-06T14:22:19.637810+0000 (oldest deadline 2024-04-06T14:22:46.038276+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:26.039026+0000 front 2024-04-06T14:22:26.038979+0000 (oldest deadline 2024-04-06T14:22:48.338831+0000) 2024-04-06T14:23:03.997 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:03 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10[78160]: 2024-04-06T14:23:03.692+0000 7f1ed4b44700 -1 osd.10 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:32.340637+0000 front 2024-04-06T14:22:32.340702+0000 (oldest deadline 2024-04-06T14:22:56.440412+0000) 2024-04-06T14:23:04.892 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:04.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:04.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:04.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:04.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:04.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:04 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:04.588+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:05.496 INFO:journalctl@ceph.osd.10.smithi105.stdout:Apr 06 14:23:05 smithi105 bash[91828]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-10 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.564+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.564+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.564+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.565+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.565+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:05.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:05 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:05.565+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:06.419 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.10.service' 2024-04-06T14:23:06.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:06.500 INFO:tasks.cephadm.osd.10:Stopped osd.10 2024-04-06T14:23:06.500 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2024-04-06T14:23:06.500 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.11 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.523+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 systemd[1]: Stopping Ceph osd.11 for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:06.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.738+0000 7f417b30b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:06.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.738+0000 7f417b30b700 -1 osd.11 107 *** Got signal Terminated *** 2024-04-06T14:23:06.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:06 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:06.738+0000 7f417b30b700 -1 osd.11 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:07.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:07 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:07.473+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:08.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:08 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:08.478+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:09.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:09 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:09.435+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.443+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.443+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.443+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.444+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.444+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.444+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:10.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:10 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:10.444+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:11.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6806 osd.0 since back 2024-04-06T14:21:48.404962+0000 front 2024-04-06T14:21:48.404836+0000 (oldest deadline 2024-04-06T14:22:13.704674+0000) 2024-04-06T14:23:11.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6804 osd.2 since back 2024-04-06T14:22:04.207246+0000 front 2024-04-06T14:22:04.207073+0000 (oldest deadline 2024-04-06T14:22:29.507313+0000) 2024-04-06T14:23:11.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6814 osd.3 since back 2024-04-06T14:22:12.908899+0000 front 2024-04-06T14:22:12.909009+0000 (oldest deadline 2024-04-06T14:22:35.208810+0000) 2024-04-06T14:23:11.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6812 osd.4 since back 2024-04-06T14:22:19.110651+0000 front 2024-04-06T14:22:19.110756+0000 (oldest deadline 2024-04-06T14:22:41.410622+0000) 2024-04-06T14:23:11.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.134:6812 osd.5 since back 2024-04-06T14:22:27.212302+0000 front 2024-04-06T14:22:27.212321+0000 (oldest deadline 2024-04-06T14:22:51.912402+0000) 2024-04-06T14:23:11.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.90:6822 osd.6 since back 2024-04-06T14:22:31.912837+0000 front 2024-04-06T14:22:31.912905+0000 (oldest deadline 2024-04-06T14:22:57.212817+0000) 2024-04-06T14:23:11.894 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:11 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11[78353]: 2024-04-06T14:23:11.412+0000 7f41738f9700 -1 osd.11 107 heartbeat_check: no reply from 172.21.15.105:6820 osd.7 since back 2024-04-06T14:22:37.213630+0000 front 2024-04-06T14:22:37.213615+0000 (oldest deadline 2024-04-06T14:23:06.014035+0000) 2024-04-06T14:23:12.893 INFO:journalctl@ceph.osd.11.smithi134.stdout:Apr 06 14:23:12 smithi134 bash[93188]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-osd-11 2024-04-06T14:23:13.658 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@osd.11.service' 2024-04-06T14:23:13.698 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:13.698 INFO:tasks.cephadm.osd.11:Stopped osd.11 2024-04-06T14:23:13.698 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-04-06T14:23:13.698 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.a 2024-04-06T14:23:13.984 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:23:13 smithi090 systemd[1]: Stopping Ceph mds.a for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:14.288 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:23:13 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-a[95199]: 2024-04-06T14:23:13.983+0000 7fc1591fd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:14.288 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:23:13 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-a[95199]: 2024-04-06T14:23:13.983+0000 7fc1591fd700 -1 mds.a *** got signal Terminated *** 2024-04-06T14:23:18.288 INFO:journalctl@ceph.mds.a.smithi090.stdout:Apr 06 14:23:18 smithi090 bash[130034]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-a 2024-04-06T14:23:18.359 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.a.service' 2024-04-06T14:23:18.397 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:18.398 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-04-06T14:23:18.398 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2024-04-06T14:23:18.398 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.d 2024-04-06T14:23:18.666 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:23:18 smithi090 systemd[1]: Stopping Ceph mds.d for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:19.038 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:23:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-d[94284]: 2024-04-06T14:23:18.666+0000 7f658794d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:19.038 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:23:18 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-d[94284]: 2024-04-06T14:23:18.666+0000 7f658794d700 -1 mds.d *** got signal Terminated *** 2024-04-06T14:23:22.208 INFO:journalctl@ceph.mds.d.smithi090.stdout:Apr 06 14:23:21 smithi090 bash[130172]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-d 2024-04-06T14:23:22.213 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.d.service' 2024-04-06T14:23:22.248 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:22.248 INFO:tasks.cephadm.mds.d:Stopped mds.d 2024-04-06T14:23:22.248 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2024-04-06T14:23:22.248 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.g 2024-04-06T14:23:22.516 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:23:22 smithi090 systemd[1]: Stopping Ceph mds.g for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:22.788 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:23:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-g[94814]: 2024-04-06T14:23:22.516+0000 7f78fee60700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:22.788 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:23:22 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-g[94814]: 2024-04-06T14:23:22.516+0000 7f78fee60700 -1 mds.g *** got signal Terminated *** 2024-04-06T14:23:24.516 INFO:journalctl@ceph.mds.g.smithi090.stdout:Apr 06 14:23:24 smithi090 bash[130311]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-g 2024-04-06T14:23:24.522 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.g.service' 2024-04-06T14:23:24.556 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:24.556 INFO:tasks.cephadm.mds.g:Stopped mds.g 2024-04-06T14:23:24.556 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2024-04-06T14:23:24.556 DEBUG:teuthology.orchestra.run.smithi090:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.j 2024-04-06T14:23:24.788 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:23:24 smithi090 systemd[1]: Stopping Ceph mds.j for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:25.288 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:23:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-j[93897]: 2024-04-06T14:23:24.824+0000 7f9946c7f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:25.288 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:23:24 smithi090 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-j[93897]: 2024-04-06T14:23:24.824+0000 7f9946c7f700 -1 mds.j *** got signal Terminated *** 2024-04-06T14:23:28.518 INFO:journalctl@ceph.mds.j.smithi090.stdout:Apr 06 14:23:28 smithi090 bash[130450]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-j 2024-04-06T14:23:28.523 DEBUG:teuthology.orchestra.run.smithi090:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.j.service' 2024-04-06T14:23:28.557 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:28.558 INFO:tasks.cephadm.mds.j:Stopped mds.j 2024-04-06T14:23:28.558 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-04-06T14:23:28.558 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.b 2024-04-06T14:23:28.996 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:23:28 smithi105 systemd[1]: Stopping Ceph mds.b for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:28.997 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:23:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: 2024-04-06T14:23:28.796+0000 7f8ef449a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:28.997 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:23:28 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b[84417]: 2024-04-06T14:23:28.796+0000 7f8ef449a700 -1 mds.b *** got signal Terminated *** 2024-04-06T14:23:33.155 INFO:journalctl@ceph.mds.b.smithi105.stdout:Apr 06 14:23:32 smithi105 bash[92082]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-b 2024-04-06T14:23:33.160 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.b.service' 2024-04-06T14:23:33.196 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:33.196 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-04-06T14:23:33.196 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2024-04-06T14:23:33.196 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.e 2024-04-06T14:23:33.453 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:23:33 smithi105 systemd[1]: Stopping Ceph mds.e for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:33.746 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:23:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: 2024-04-06T14:23:33.452+0000 7fc431726700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:33.747 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:23:33 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e[83724]: 2024-04-06T14:23:33.452+0000 7fc431726700 -1 mds.e *** got signal Terminated *** 2024-04-06T14:23:37.864 INFO:journalctl@ceph.mds.e.smithi105.stdout:Apr 06 14:23:37 smithi105 bash[92222]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-e 2024-04-06T14:23:37.869 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.e.service' 2024-04-06T14:23:37.903 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:37.904 INFO:tasks.cephadm.mds.e:Stopped mds.e 2024-04-06T14:23:37.904 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2024-04-06T14:23:37.904 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.h 2024-04-06T14:23:38.152 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:23:37 smithi105 systemd[1]: Stopping Ceph mds.h for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:38.496 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:23:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: 2024-04-06T14:23:38.152+0000 7fc9f0451700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:38.497 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:23:38 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h[84074]: 2024-04-06T14:23:38.152+0000 7fc9f0451700 -1 mds.h *** got signal Terminated *** 2024-04-06T14:23:42.126 INFO:journalctl@ceph.mds.h.smithi105.stdout:Apr 06 14:23:41 smithi105 bash[92359]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-h 2024-04-06T14:23:42.132 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.h.service' 2024-04-06T14:23:42.164 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:42.165 INFO:tasks.cephadm.mds.h:Stopped mds.h 2024-04-06T14:23:42.165 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2024-04-06T14:23:42.165 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.k 2024-04-06T14:23:42.419 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:23:42 smithi105 systemd[1]: Stopping Ceph mds.k for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:42.746 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:23:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-k[83385]: 2024-04-06T14:23:42.418+0000 7fd51b93a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:42.747 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:23:42 smithi105 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-k[83385]: 2024-04-06T14:23:42.418+0000 7fd51b93a700 -1 mds.k *** got signal Terminated *** 2024-04-06T14:23:46.428 INFO:journalctl@ceph.mds.k.smithi105.stdout:Apr 06 14:23:46 smithi105 bash[92497]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-k 2024-04-06T14:23:46.434 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.k.service' 2024-04-06T14:23:46.469 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:46.469 INFO:tasks.cephadm.mds.k:Stopped mds.k 2024-04-06T14:23:46.469 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2024-04-06T14:23:46.470 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.c 2024-04-06T14:23:46.893 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:23:46 smithi134 systemd[1]: Stopping Ceph mds.c for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:46.893 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:23:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: 2024-04-06T14:23:46.715+0000 7f1467102700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:46.893 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:23:46 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c[85046]: 2024-04-06T14:23:46.715+0000 7f1467102700 -1 mds.c *** got signal Terminated *** 2024-04-06T14:23:50.142 INFO:journalctl@ceph.mds.c.smithi134.stdout:Apr 06 14:23:49 smithi134 bash[93451]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-c 2024-04-06T14:23:50.234 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.c.service' 2024-04-06T14:23:50.272 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:50.272 INFO:tasks.cephadm.mds.c:Stopped mds.c 2024-04-06T14:23:50.272 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2024-04-06T14:23:50.272 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.f 2024-04-06T14:23:50.531 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:23:50 smithi134 systemd[1]: Stopping Ceph mds.f for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:50.893 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:23:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: 2024-04-06T14:23:50.531+0000 7f4c9b5f6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:50.893 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:23:50 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f[84696]: 2024-04-06T14:23:50.531+0000 7f4c9b5f6700 -1 mds.f *** got signal Terminated *** 2024-04-06T14:23:54.142 INFO:journalctl@ceph.mds.f.smithi134.stdout:Apr 06 14:23:53 smithi134 bash[93590]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-f 2024-04-06T14:23:54.164 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.f.service' 2024-04-06T14:23:54.198 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:54.199 INFO:tasks.cephadm.mds.f:Stopped mds.f 2024-04-06T14:23:54.199 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2024-04-06T14:23:54.199 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.i 2024-04-06T14:23:54.455 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:23:54 smithi134 systemd[1]: Stopping Ceph mds.i for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:54.892 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:23:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: 2024-04-06T14:23:54.455+0000 7f3f73edd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:54.893 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:23:54 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i[85389]: 2024-04-06T14:23:54.455+0000 7f3f73edd700 -1 mds.i *** got signal Terminated *** 2024-04-06T14:23:58.108 INFO:journalctl@ceph.mds.i.smithi134.stdout:Apr 06 14:23:57 smithi134 bash[93727]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-i 2024-04-06T14:23:58.114 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.i.service' 2024-04-06T14:23:58.147 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:23:58.147 INFO:tasks.cephadm.mds.i:Stopped mds.i 2024-04-06T14:23:58.147 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2024-04-06T14:23:58.147 DEBUG:teuthology.orchestra.run.smithi134:> sudo systemctl stop ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.l 2024-04-06T14:23:58.392 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:23:58 smithi134 systemd[1]: Stopping Ceph mds.l for 949dca1a-f41d-11ee-b649-cb9ed24678a4... 2024-04-06T14:23:58.893 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:23:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-l[84356]: 2024-04-06T14:23:58.412+0000 7f4e5df09700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-06T14:23:58.893 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:23:58 smithi134 ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-l[84356]: 2024-04-06T14:23:58.412+0000 7f4e5df09700 -1 mds.l *** got signal Terminated *** 2024-04-06T14:24:00.620 INFO:journalctl@ceph.mds.l.smithi134.stdout:Apr 06 14:24:00 smithi134 bash[93865]: ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4-mds-l 2024-04-06T14:24:00.627 DEBUG:teuthology.orchestra.run.smithi134:> sudo pkill -f 'journalctl -f -n 0 -u ceph-949dca1a-f41d-11ee-b649-cb9ed24678a4@mds.l.service' 2024-04-06T14:24:00.660 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-06T14:24:00.661 INFO:tasks.cephadm.mds.l:Stopped mds.l 2024-04-06T14:24:00.661 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-06T14:24:00.845 INFO:teuthology.orchestra.run.smithi090.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:24:02.326 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-06T14:24:02.570 INFO:teuthology.orchestra.run.smithi105.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:24:04.011 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-06T14:24:04.241 INFO:teuthology.orchestra.run.smithi134.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:24:05.755 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:24:05.792 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:24:05.828 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-06T14:24:05.859 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-06T14:24:05.861 DEBUG:teuthology.misc:Transferring archived files from smithi090:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi090/crash 2024-04-06T14:24:05.861 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash -- . 2024-04-06T14:24:05.919 INFO:teuthology.orchestra.run.smithi090.stderr:tar: /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-06T14:24:05.919 INFO:teuthology.orchestra.run.smithi090.stderr:tar: Error is not recoverable: exiting now 2024-04-06T14:24:05.921 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi105/crash 2024-04-06T14:24:05.922 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash -- . 2024-04-06T14:24:05.982 INFO:teuthology.orchestra.run.smithi105.stderr:tar: /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-06T14:24:05.982 INFO:teuthology.orchestra.run.smithi105.stderr:tar: Error is not recoverable: exiting now 2024-04-06T14:24:05.984 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi134/crash 2024-04-06T14:24:05.984 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash -- . 2024-04-06T14:24:06.067 INFO:teuthology.orchestra.run.smithi134.stderr:tar: /var/lib/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-06T14:24:06.067 INFO:teuthology.orchestra.run.smithi134.stderr:tar: Error is not recoverable: exiting now 2024-04-06T14:24:06.068 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-06T14:24:06.068 DEBUG:teuthology.orchestra.run.smithi090:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v 'slow metadata IO' | egrep -v SLOW_OPS | egrep -v 'slow request' | head -n 1 2024-04-06T14:24:06.104 INFO:teuthology.orchestra.run.smithi090.stderr:grep: /var/log/ceph/949dca1a-f41d-11ee-b649-cb9ed24678a4/ceph.log: No such file or directory 2024-04-06T14:24:06.107 INFO:tasks.cephadm:Compressing logs... 2024-04-06T14:24:06.107 DEBUG:teuthology.orchestra.run.smithi090:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:24:06.149 DEBUG:teuthology.orchestra.run.smithi105:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:24:06.152 DEBUG:teuthology.orchestra.run.smithi134:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:24:06.192 INFO:teuthology.orchestra.run.smithi105.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-06T14:24:06.192 INFO:teuthology.orchestra.run.smithi090.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-06T14:24:06.200 INFO:teuthology.orchestra.run.smithi134.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-06T14:29:20.841 INFO:tasks.cephadm:Archiving logs... 2024-04-06T14:29:20.842 DEBUG:teuthology.misc:Transferring archived files from smithi090:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi090/log 2024-04-06T14:29:20.843 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-06T14:30:01.569 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi105/log 2024-04-06T14:30:01.570 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-06T14:30:44.603 DEBUG:teuthology.misc:Transferring archived files from smithi134:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi134/log 2024-04-06T14:30:44.605 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-06T14:31:29.903 INFO:tasks.cephadm:Removing cluster... 2024-04-06T14:31:29.903 DEBUG:teuthology.orchestra.run.smithi090:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force 2024-04-06T14:31:30.098 INFO:teuthology.orchestra.run.smithi090.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:31:30.649 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force 2024-04-06T14:31:30.838 INFO:teuthology.orchestra.run.smithi105.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:31:31.389 DEBUG:teuthology.orchestra.run.smithi134:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 949dca1a-f41d-11ee-b649-cb9ed24678a4 --force 2024-04-06T14:31:31.580 INFO:teuthology.orchestra.run.smithi134.stdout:Deleting cluster with fsid: 949dca1a-f41d-11ee-b649-cb9ed24678a4 2024-04-06T14:31:32.160 INFO:tasks.cephadm:Removing cephadm ... 2024-04-06T14:31:32.161 DEBUG:teuthology.orchestra.run.smithi090:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-06T14:31:32.182 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-06T14:31:32.302 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-06T14:31:32.334 INFO:tasks.cephadm:Teardown complete 2024-04-06T14:31:32.335 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-06T14:31:32.367 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-06T14:31:32.367 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-06T14:31:32.370 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-06T14:31:32.372 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-06T14:31:32.493 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-06T14:31:32.508 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-06T14:31:32.508 DEBUG:teuthology.orchestra.run.smithi090:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T14:31:32.511 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T14:31:32.514 DEBUG:teuthology.orchestra.run.smithi134:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-06T14:31:32.527 INFO:teuthology.orchestra.run.smithi090.stderr:bash: ntpq: command not found 2024-04-06T14:31:32.530 INFO:teuthology.orchestra.run.smithi134.stderr:bash: ntpq: command not found 2024-04-06T14:31:32.530 INFO:teuthology.orchestra.run.smithi105.stderr:bash: ntpq: command not found 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:=============================================================================== 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 19 -778us[ -788us] +/- 94ms 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:^* hv02.front.sepia.ceph.com 3 6 377 19 +558us[ +548us] +/- 53ms 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 20 -172us[ -182us] +/- 82ms 2024-04-06T14:31:32.532 INFO:teuthology.orchestra.run.smithi090.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T14:31:32.546 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T14:31:32.547 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-04-06T14:31:32.547 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 18 -798us[ -798us] +/- 94ms 2024-04-06T14:31:32.547 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv02.front.sepia.ceph.com 3 6 377 18 +556us[ +565us] +/- 54ms 2024-04-06T14:31:32.547 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 -169us[ -159us] +/- 82ms 2024-04-06T14:31:32.547 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T14:31:32.554 INFO:teuthology.orchestra.run.smithi134.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-06T14:31:32.555 INFO:teuthology.orchestra.run.smithi134.stdout:=============================================================================== 2024-04-06T14:31:32.555 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 16 -770us[ -770us] +/- 94ms 2024-04-06T14:31:32.555 INFO:teuthology.orchestra.run.smithi134.stdout:^* hv02.front.sepia.ceph.com 3 6 377 18 +575us[ +580us] +/- 54ms 2024-04-06T14:31:32.555 INFO:teuthology.orchestra.run.smithi134.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 15 -172us[ -172us] +/- 82ms 2024-04-06T14:31:32.555 INFO:teuthology.orchestra.run.smithi134.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-06T14:31:32.556 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-06T14:31:32.569 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-06T14:31:32.570 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-06T14:31:32.583 DEBUG:teuthology.orchestra.run.smithi090:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-06T14:31:32.694 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-06T14:31:32.798 DEBUG:teuthology.orchestra.run.smithi134:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-06T14:31:32.875 DEBUG:teuthology.orchestra.run.smithi090:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T14:31:32.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:32.909 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T14:31:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:32.943 DEBUG:teuthology.orchestra.run.smithi134:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-06T14:31:32.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:32.973 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-06T14:31:32.985 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-06T14:31:32.996 INFO:teuthology.task.internal:Duration was 2869.076965 seconds 2024-04-06T14:31:32.996 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-06T14:31:33.008 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-06T14:31:33.008 DEBUG:teuthology.orchestra.run.smithi090:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-06T14:31:33.011 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-06T14:31:33.013 DEBUG:teuthology.orchestra.run.smithi134:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-06T14:31:33.085 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T14:31:33.090 INFO:teuthology.orchestra.run.smithi090.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T14:31:33.169 INFO:teuthology.orchestra.run.smithi134.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-06T14:31:33.527 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-06T14:31:33.527 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi090.front.sepia.ceph.com 2024-04-06T14:31:33.527 DEBUG:teuthology.orchestra.run.smithi090:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-06T14:31:33.554 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-04-06T14:31:33.555 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-06T14:31:33.581 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi134.front.sepia.ceph.com 2024-04-06T14:31:33.582 DEBUG:teuthology.orchestra.run.smithi134:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-06T14:31:33.607 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-06T14:31:33.607 DEBUG:teuthology.orchestra.run.smithi090:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:31:33.610 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:31:33.624 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-06T14:31:33.703 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-06T14:31:33.703 DEBUG:teuthology.orchestra.run.smithi090:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-06T14:31:33.705 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-06T14:31:33.708 DEBUG:teuthology.orchestra.run.smithi134:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-06T14:31:33.820 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-06T14:31:33.843 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-06T14:31:33.843 DEBUG:teuthology.orchestra.run.smithi090:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-06T14:31:33.896 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-06T14:31:33.940 DEBUG:teuthology.orchestra.run.smithi134:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-06T14:31:33.973 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-06T14:31:33.985 DEBUG:teuthology.orchestra.run.smithi090:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:33.988 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:33.990 DEBUG:teuthology.orchestra.run.smithi134:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:34.017 INFO:teuthology.orchestra.run.smithi090.stdout:kernel.core_pattern = core 2024-04-06T14:31:34.029 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-04-06T14:31:34.046 INFO:teuthology.orchestra.run.smithi134.stdout:kernel.core_pattern = core 2024-04-06T14:31:34.070 DEBUG:teuthology.orchestra.run.smithi090:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:34.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:34.098 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:34.115 DEBUG:teuthology.orchestra.run.smithi134:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-06T14:31:34.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-06T14:31:34.132 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-06T14:31:34.147 INFO:teuthology.task.internal:Transferring archived files... 2024-04-06T14:31:34.149 DEBUG:teuthology.misc:Transferring archived files from smithi090:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi090 2024-04-06T14:31:34.149 DEBUG:teuthology.orchestra.run.smithi090:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-06T14:31:34.193 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi105 2024-04-06T14:31:34.194 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-06T14:31:34.235 DEBUG:teuthology.misc:Transferring archived files from smithi134:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056/remote/smithi134 2024-04-06T14:31:34.236 DEBUG:teuthology.orchestra.run.smithi134:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-06T14:31:34.276 INFO:teuthology.task.internal:Removing archive directory... 2024-04-06T14:31:34.276 DEBUG:teuthology.orchestra.run.smithi090:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-06T14:31:34.279 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-06T14:31:34.281 DEBUG:teuthology.orchestra.run.smithi134:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-06T14:31:34.327 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-06T14:31:34.341 INFO:teuthology.task.internal:Not uploading archives. 2024-04-06T14:31:34.341 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-06T14:31:34.353 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-06T14:31:34.353 DEBUG:teuthology.orchestra.run.smithi090:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-06T14:31:34.355 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-06T14:31:34.357 DEBUG:teuthology.orchestra.run.smithi134:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-06T14:31:34.370 INFO:teuthology.orchestra.run.smithi090.stdout: 262175 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 6 14:31 /home/ubuntu/cephtest 2024-04-06T14:31:34.372 INFO:teuthology.orchestra.run.smithi105.stdout: 262157 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 6 14:31 /home/ubuntu/cephtest 2024-04-06T14:31:34.384 INFO:teuthology.orchestra.run.smithi134.stdout: 262157 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 6 14:31 /home/ubuntu/cephtest 2024-04-06T14:31:34.385 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-06T14:31:34.397 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-06T14:31:34.422 INFO:teuthology.nuke:Checking targets against current locks 2024-04-06T14:31:34.449 DEBUG:teuthology.nuke:shortname: smithi090 2024-04-06T14:31:34.449 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-06T14:31:34.471 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi090.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.883055', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLdBdDgbEv91YZhuMvZHDVA25oXplN4DZaTR0BcGXPTRTTRwHpuZOLEBiSIb22QDGpB+GE6NufEmYxEUeTEudiA='} 2024-04-06T14:31:34.476 DEBUG:teuthology.nuke:shortname: smithi105 2024-04-06T14:31:34.476 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-06T14:31:34.500 DEBUG:teuthology.nuke:shortname: smithi134 2024-04-06T14:31:34.500 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-06T14:31:34.504 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.886477', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOVFhnVzOckaGR4At43MeDqaE0SvMXg0YOZWW1569JXLfOcYV4LD1S3UuRsg7qYW6mPSdbvhTv4JZTtwwc4i+vE='} 2024-04-06T14:31:34.535 INFO:teuthology.orchestra.console.smithi090:Power off 2024-04-06T14:31:34.535 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-06T14:31:34.545 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi134.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-03_22:24:02-fs-reef-distro-default-smithi/7639056', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-06 13:37:29.884901', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAkp91UjlV6CQ+U6xQnPXdnsnfn6Q9zAvre4oI9Zy5N78VmFK+CIHoQCM6YR831rdVHlu58dd4ol3KjVRIPMr1s='} 2024-04-06T14:31:34.558 DEBUG:teuthology.orchestra.console.smithi090:power off output: Chassis Power Control: Down/Off 2024-04-06T14:31:34.558 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:34.576 INFO:teuthology.orchestra.console.smithi105:Power off 2024-04-06T14:31:34.576 DEBUG:teuthology.orchestra.console.smithi105:pexpect command: ipmitool -H smithi105.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-06T14:31:34.591 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is on 2024-04-06T14:31:34.599 DEBUG:teuthology.orchestra.console.smithi105:power off output: Chassis Power Control: Down/Off 2024-04-06T14:31:34.599 DEBUG:teuthology.orchestra.console.smithi105:pexpect command: ipmitool -H smithi105.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:34.619 INFO:teuthology.orchestra.console.smithi134:Power off 2024-04-06T14:31:34.619 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-06T14:31:34.628 DEBUG:teuthology.orchestra.console.smithi105:check power output: Chassis Power is on 2024-04-06T14:31:34.641 DEBUG:teuthology.orchestra.console.smithi134:power off output: Chassis Power Control: Down/Off 2024-04-06T14:31:34.642 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:34.666 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2024-04-06T14:31:38.593 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:38.629 DEBUG:teuthology.orchestra.console.smithi105:pexpect command: ipmitool -H smithi105.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:38.667 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:38.703 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is on 2024-04-06T14:31:38.740 DEBUG:teuthology.orchestra.console.smithi105:check power output: Chassis Power is on 2024-04-06T14:31:38.778 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is on 2024-04-06T14:31:42.706 DEBUG:teuthology.orchestra.console.smithi090:pexpect command: ipmitool -H smithi090.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:42.741 DEBUG:teuthology.orchestra.console.smithi105:pexpect command: ipmitool -H smithi105.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:42.779 DEBUG:teuthology.orchestra.console.smithi134:pexpect command: ipmitool -H smithi134.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-06T14:31:42.817 DEBUG:teuthology.orchestra.console.smithi090:check power output: Chassis Power is off 2024-04-06T14:31:42.851 DEBUG:teuthology.orchestra.console.smithi105:check power output: Chassis Power is off 2024-04-06T14:31:42.890 DEBUG:teuthology.orchestra.console.smithi134:check power output: Chassis Power is off 2024-04-06T14:31:42.918 INFO:teuthology.orchestra.console.smithi090:Power off completed 2024-04-06T14:31:42.952 INFO:teuthology.orchestra.console.smithi105:Power off completed 2024-04-06T14:31:42.991 INFO:teuthology.orchestra.console.smithi134:Power off completed 2024-04-06T14:31:43.096 INFO:teuthology.run:Summary data: description: fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate 3-modules} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/secure wsync/yes} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{balancer/random export-check n/3 replication/always} standby-replay tasks/{0-subvolume/{with-namespace-isolated-and-quota} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} duration: 2869.076964855194 failure_reason: 'The following counters failed to be set on mds daemons: {''mds.exported'', ''mds.imported''}' flavor: default owner: scheduled_teuthology@teuthology status: fail success: false 2024-04-06T14:31:43.097 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-06T14:31:43.191 INFO:teuthology.run:FAIL